Make the parser set a span on each AST node.

This commit is contained in:
Simon Binder 2019-09-18 21:36:25 +02:00
parent 1bd856e9c5
commit 1bdfa0289b
No known key found for this signature in database
GPG Key ID: 7891917E4147B8C0
5 changed files with 58 additions and 18 deletions

View File

@ -274,6 +274,7 @@ mixin CrudParser on ParserBase {
OrderByBase _orderBy() {
if (_matchOne(TokenType.order)) {
final orderToken = _previous;
_consume(TokenType.by, 'Expected "BY" after "ORDER" token');
final terms = <OrderingTermBase>[];
do {
@ -290,7 +291,7 @@ mixin CrudParser on ParserBase {
..setSpan(termPlaceholder.first, termPlaceholder.last);
}
return OrderBy(terms: terms);
return OrderBy(terms: terms)..setSpan(orderToken, _previous);
}
return null;
}
@ -307,7 +308,8 @@ mixin CrudParser on ParserBase {
..setSpan(expr.first, expr.last);
}
return OrderingTerm(expression: expr, orderingMode: mode);
return OrderingTerm(expression: expr, orderingMode: mode)
..setSpan(expr.first, _previous);
}
@override
@ -356,6 +358,8 @@ mixin CrudParser on ParserBase {
DeleteStatement _deleteStmt() {
if (!_matchOne(TokenType.delete)) return null;
final deleteToken = _previous;
_consume(TokenType.from, 'Expected a FROM here');
final table = _tableReference();
@ -368,7 +372,8 @@ mixin CrudParser on ParserBase {
where = expression();
}
return DeleteStatement(from: table, where: where);
return DeleteStatement(from: table, where: where)
..setSpan(deleteToken, _previous);
}
UpdateStatement _update() {
@ -393,7 +398,8 @@ mixin CrudParser on ParserBase {
_consume(TokenType.equal, 'Expected = after the column name');
final expr = expression();
set.add(SetComponent(column: reference, expression: expr));
set.add(SetComponent(column: reference, expression: expr)
..setSpan(columnName, _previous));
} while (_matchOne(TokenType.comma));
final where = _where();
@ -428,7 +434,7 @@ mixin CrudParser on ParserBase {
insertMode = InsertMode.insert;
}
} else {
// is it wasn't an insert, it must have been a replace
// if it wasn't an insert, it must have been a replace
insertMode = InsertMode.replace;
}
assert(insertMode != null);
@ -440,7 +446,8 @@ mixin CrudParser on ParserBase {
if (_matchOne(TokenType.leftParen)) {
do {
final columnRef = _consumeIdentifier('Expected a column');
targetColumns.add(Reference(columnName: columnRef.identifier));
targetColumns.add(Reference(columnName: columnRef.identifier)
..setSpan(columnRef, columnRef));
} while (_matchOne(TokenType.comma));
_consume(TokenType.rightParen,

View File

@ -20,15 +20,20 @@ mixin ExpressionParser on ParserBase {
Expression _case() {
if (_matchOne(TokenType.$case)) {
final caseToken = _previous;
final base = _check(TokenType.when) ? null : _or();
final whens = <WhenComponent>[];
Expression $else;
while (_matchOne(TokenType.when)) {
final whenToken = _previous;
final whenExpr = _or();
_consume(TokenType.then, 'Expected THEN');
final then = _or();
whens.add(WhenComponent(when: whenExpr, then: then));
whens.add(WhenComponent(when: whenExpr, then: then)
..setSpan(whenToken, _previous));
}
if (_matchOne(TokenType.$else)) {
@ -36,7 +41,8 @@ mixin ExpressionParser on ParserBase {
}
_consume(TokenType.end, 'Expected END to finish the case operator');
return CaseExpression(whens: whens, base: base, elseExpr: $else);
return CaseExpression(whens: whens, base: base, elseExpr: $else)
..setSpan(caseToken, _previous);
}
return _or();
@ -52,7 +58,8 @@ mixin ExpressionParser on ParserBase {
while (_match(types)) {
final operator = _previous;
final right = higherPrecedence();
expression = BinaryExpression(expression, operator, right);
expression = BinaryExpression(expression, operator, right)
..setSpan(expression.first, _previous);
}
return expression;
}
@ -68,7 +75,8 @@ mixin ExpressionParser on ParserBase {
_matchOne(TokenType.$in);
final inside = _variableOrNull() ?? _consumeTuple(orSubQuery: true);
return InExpression(left: left, inside: inside, not: not);
return InExpression(left: left, inside: inside, not: not)
..setSpan(left.first, _previous);
}
return left;
@ -79,6 +87,7 @@ mixin ExpressionParser on ParserBase {
/// expressions.
Expression _equals() {
var expression = _comparison();
final first = expression.first;
final ops = const [
TokenType.equal,
@ -104,15 +113,18 @@ mixin ExpressionParser on ParserBase {
final upper = _comparison();
expression = BetweenExpression(
not: not, check: expression, lower: lower, upper: upper);
not: not, check: expression, lower: lower, upper: upper)
..setSpan(first, _previous);
} else if (_match(ops)) {
final operator = _previous;
if (operator.type == TokenType.$is) {
final not = _match(const [TokenType.not]);
// special case: is not expression
expression = IsExpression(not, expression, _comparison());
expression = IsExpression(not, expression, _comparison())
..setSpan(first, _previous);
} else {
expression = BinaryExpression(expression, operator, _comparison());
expression = BinaryExpression(expression, operator, _comparison())
..setSpan(first, _previous);
}
} else if (_checkAnyWithNot(stringOps)) {
final not = _matchOne(TokenType.not);
@ -130,7 +142,8 @@ mixin ExpressionParser on ParserBase {
left: expression,
operator: operator,
right: right,
escape: escape);
escape: escape)
..setSpan(first, _previous);
} else {
break; // no matching operator with this precedence was found
}
@ -175,14 +188,17 @@ mixin ExpressionParser on ParserBase {
])) {
final operator = _previous;
final expression = _unary();
return UnaryExpression(operator, expression);
return UnaryExpression(operator, expression)
..setSpan(operator, expression.last);
} else if (_matchOne(TokenType.exists)) {
final existsToken = _previous;
_consume(
TokenType.leftParen, 'Expected opening parenthesis after EXISTS');
final selectStmt = select();
_consume(TokenType.rightParen,
'Expected closing paranthesis to finish EXISTS expression');
return ExistsExpression(select: selectStmt);
return ExistsExpression(select: selectStmt)
..setSpan(existsToken, _previous);
}
return _postfix();
@ -255,11 +271,11 @@ mixin ExpressionParser on ParserBase {
if (_peek.type == TokenType.select) {
final stmt = select();
_consume(TokenType.rightParen, 'Expected a closing bracket');
return SubQuery(select: stmt);
return SubQuery(select: stmt)..setSpan(left, _previous);
} else {
final expr = expression();
_consume(TokenType.rightParen, 'Expected a closing bracket');
return Parentheses(left, expr, token);
return Parentheses(left, expr, token)..setSpan(left, _previous);
}
break;
case TokenType.identifier:

View File

@ -146,6 +146,8 @@ void main() {
final tokens = scanner.scanTokens();
final parser = Parser(tokens);
final expression = parser.expression();
enforceHasSpan(expression);
enforceEqual(expression, expected);
});
});

View File

@ -2,6 +2,8 @@ import 'package:sqlparser/sqlparser.dart';
import 'package:sqlparser/src/utils/ast_equality.dart';
import 'package:test/test.dart';
import 'utils.dart';
const content = r'''
import 'other.dart';
import 'another.moor';
@ -19,6 +21,7 @@ void main() {
test('parses moor files', () {
final parsed = SqlEngine(useMoorExtensions: true).parseMoorFile(content);
final file = parsed.rootNode;
enforceHasSpan(file);
enforceEqual(
file,

View File

@ -20,6 +20,7 @@ IdentifierToken identifier(String content) {
void testStatement(String sql, AstNode expected, {bool moorMode = false}) {
final parsed = SqlEngine(useMoorExtensions: moorMode).parse(sql).rootNode;
enforceHasSpan(parsed);
enforceEqual(parsed, expected);
}
@ -34,3 +35,14 @@ void testAll(Map<String, AstNode> testCases) {
});
});
}
/// The parser should make sure [AstNode.hasSpan] is true on relevant nodes.
void enforceHasSpan(AstNode node) {
final problematic = [node]
.followedBy(node.allDescendants)
.firstWhere((node) => !node.hasSpan, orElse: () => null);
if (problematic != null) {
throw ArgumentError('Node $problematic did not have a span');
}
}