Improve variable tokenization for easier parsing logic

sqlparser/lib/src/ast/ast.dart

@@ -23,6 +23,7 @@ part 'expressions/variables.dart';
 
 part 'moor/declared_statement.dart';
 part 'moor/import_statement.dart';
+part 'moor/inline_dart.dart';
 part 'moor/moor_file.dart';
 
 part 'schema/column_definition.dart';
@@ -191,6 +192,7 @@ abstract class AstVisitor<T> {
   T visitMoorFile(MoorFile e);
   T visitMoorImportStatement(ImportStatement e);
   T visitMoorDeclaredStatement(DeclaredStatement e);
+  T visitInlineDartCode(InlineDart e);
 }
 
 /// Visitor that walks down the entire tree, visiting all children in order.
@@ -312,6 +314,9 @@ class RecursiveVisitor<T> extends AstVisitor<T> {
   @override
   T visitMoorDeclaredStatement(DeclaredStatement e) => visitChildren(e);
 
+  @override
+  T visitInlineDartCode(InlineDart e) => visitChildren(e);
+
   @protected
   T visitChildren(AstNode e) {
     for (var child in e.childNodes) {

sqlparser/lib/src/ast/clauses/limit.dart

@@ -1,6 +1,8 @@
 part of '../ast.dart';
 
-class Limit extends AstNode {
+abstract class LimitBase {}
+
+class Limit extends AstNode implements LimitBase {
   Expression count;
   Token offsetSeparator; // can either be OFFSET or just a comma
   Expression offset;

sqlparser/lib/src/ast/expressions/variables.dart

@@ -6,10 +6,10 @@ mixin Variable on Expression {
 
 /// A "?" or "?123" variable placeholder
 class NumberedVariable extends Expression with Variable {
-  final Token questionMark;
-  final int explicitIndex;
+  final QuestionMarkVariableToken token;
+  int get explicitIndex => token.explicitIndex;
 
-  NumberedVariable(this.questionMark, this.explicitIndex);
+  NumberedVariable(this.token);
 
   @override
   T accept<T>(AstVisitor<T> visitor) {
@@ -26,9 +26,10 @@ class NumberedVariable extends Expression with Variable {
 }
 
 class ColonNamedVariable extends Expression with Variable {
-  final String name;
+  final ColonVariableToken token;
+  String get name => token.name;
 
-  ColonNamedVariable(this.name);
+  ColonNamedVariable(this.token);
 
   @override
   T accept<T>(AstVisitor<T> visitor) {

sqlparser/lib/src/ast/moor/inline_dart.dart

@@ -0,0 +1,47 @@
+part of '../ast.dart';
+
+/// An inline Dart component that appears in a compiled sql query. Inline Dart
+/// components can be bound with complex expressions at runtime by using moor's
+/// Dart API.
+///
+/// At the moment, we support 4 kind of inline components:
+///  1. expressions: Any expression can be used for moor: `SELECT * FROM table
+///  = $expr`. Generated code will write this as an `Expression` class from
+///  moor.
+///  2. limits
+///  3. A single order-by clause
+///  4. A list of order-by clauses
+abstract class InlineDart extends AstNode {
+  final String name;
+
+  DollarSignVariableToken token;
+
+  InlineDart._(this.name);
+
+  @override
+  final Iterable<AstNode> childNodes = const Iterable.empty();
+
+  @override
+  T accept<T>(AstVisitor<T> visitor) => visitor.visitInlineDartCode(this);
+
+  bool _dartEquals(covariant InlineDart other);
+
+  @override
+  bool contentEquals(InlineDart other) {
+    return other.name == name && other._dartEquals(other);
+  }
+}
+
+class InlineDartExpression extends InlineDart implements Expression {
+  InlineDartExpression({@required String name}) : super._(name);
+
+  @override
+  bool _dartEquals(InlineDartExpression other) => true;
+}
+
+class InlineDartLimit extends InlineDart implements LimitBase {
+  InlineDartLimit({@required String name}) : super._(name);
+
+  @override
+  bool _dartEquals(InlineDartLimit other) => true;
+}

sqlparser/lib/src/reader/parser/expressions.dart

@@ -313,25 +313,20 @@ mixin ExpressionParser on ParserBase {
           return Reference(columnName: first.identifier)..setSpan(first, first);
         }
         break;
-      case TokenType.questionMark:
-        final mark = token;
-
-        if (_matchOne(TokenType.numberLiteral)) {
-          final number = _previous;
-          return NumberedVariable(mark, _parseNumber(number.lexeme).toInt())
-            ..setSpan(mark, number);
-        } else {
-          return NumberedVariable(mark, null)..setSpan(mark, mark);
+      case TokenType.questionMarkVariable:
+        return NumberedVariable(token as QuestionMarkVariableToken)
+          ..setSpan(token, token);
+      case TokenType.colonVariable:
+        return ColonNamedVariable(token as ColonVariableToken)
+          ..setSpan(token, token);
+      case TokenType.dollarSignVariable:
+        if (enableMoorExtensions) {
+          final typedToken = token as DollarSignVariableToken;
+          return InlineDartExpression(name: typedToken.name)
+            ..token = typedToken
+            ..setSpan(token, token);
         }
         break;
-      case TokenType.colon:
-        final colon = token;
-        final identifier = _consumeIdentifier(
-            'Expected an identifier for the named variable',
-            lenient: true);
-
-        final content = identifier.identifier;
-        return ColonNamedVariable(':$content')..setSpan(colon, identifier);
       default:
         break;
     }

sqlparser/lib/src/reader/tokenizer/scanner.dart

@@ -112,10 +112,31 @@ class Scanner {
         break;
 
       case '?':
-        _addToken(TokenType.questionMark);
+        // if the next chars are numbers, this is an explicitly index variable
+        final buffer = StringBuffer();
+        while (!_isAtEnd && isDigit(_peek())) {
+          buffer.write(_nextChar());
+        }
+
+        int explicitIndex;
+        if (buffer.isNotEmpty) {
+          explicitIndex = int.parse(buffer.toString());
+        }
+
+        tokens.add(QuestionMarkVariableToken(_currentSpan, explicitIndex));
         break;
       case ':':
-        _addToken(TokenType.colon);
+        final name = _matchColumnName();
+        if (name == null) {
+          _addToken(TokenType.colon);
+        } else {
+          tokens.add(ColonVariableToken(_currentSpan, ':$name'));
+        }
+
+        break;
+      case r'$':
+        final name = _matchColumnName();
+        tokens.add(ColonVariableToken(_currentSpan, name));
         break;
       case ';':
         _addToken(TokenType.semicolon);
@@ -330,6 +351,17 @@ class Scanner {
     }
   }
 
+  String _matchColumnName() {
+    if (_isAtEnd || !canStartColumnName(_peek())) return null;
+
+    final buffer = StringBuffer()..write(_nextChar());
+    while (!_isAtEnd && continuesColumnName(_peek())) {
+      buffer.write(_nextChar());
+    }
+
+    return buffer.toString();
+  }
+
   void _inlineDart() {
     // inline starts with a `, we just need to find the matching ` that
     // terminates this token.

sqlparser/lib/src/reader/tokenizer/token.dart

@@ -38,11 +38,12 @@ enum TokenType {
   exists,
   collate,
 
-  questionMark,
+  questionMarkVariable,
   colon,
-  // todo at and dollarSign are currently not used
+  colonVariable,
+  // todo at is not used at the moment
   at,
-  dollarSign,
+  dollarSignVariable,
 
   stringLiteral,
   numberLiteral,
@@ -293,6 +294,33 @@ class IdentifierToken extends Token {
       : super(TokenType.identifier, span);
 }
 
+abstract class VariableToken extends Token {
+  VariableToken(TokenType type, FileSpan span) : super(type, span);
+}
+
+class QuestionMarkVariableToken extends Token {
+  /// The explicit index, if this variable was of the form `?123`. Otherwise
+  /// null.
+  final int explicitIndex;
+
+  QuestionMarkVariableToken(FileSpan span, this.explicitIndex)
+      : super(TokenType.questionMarkVariable, span);
+}
+
+class ColonVariableToken extends Token {
+  final String name;
+
+  ColonVariableToken(FileSpan span, this.name)
+      : super(TokenType.colonVariable, span);
+}
+
+class DollarSignVariableToken extends Token {
+  final String name;
+
+  DollarSignVariableToken(FileSpan span, this.name)
+      : super(TokenType.dollarSignVariable, span);
+}
+
 /// Inline Dart appearing in a create table statement. Only parsed when the moor
 /// extensions are enabled. Dart code is wrapped in backticks.
 class InlineDartToken extends Token {

sqlparser/test/parser/expression_test.dart

@@ -41,15 +41,15 @@ final Map<String, Expression> _testCases = {
   '? * ?3 + ?2 == :test': BinaryExpression(
     BinaryExpression(
       BinaryExpression(
-        NumberedVariable(token(TokenType.questionMark), null),
+        NumberedVariable(QuestionMarkVariableToken(fakeSpan('?'), null)),
         token(TokenType.star),
-        NumberedVariable(token(TokenType.questionMark), 3),
+        NumberedVariable(QuestionMarkVariableToken(fakeSpan('?3'), 3)),
       ),
       token(TokenType.plus),
-      NumberedVariable(token(TokenType.questionMark), 2),
+      NumberedVariable(QuestionMarkVariableToken(fakeSpan('?2'), 2)),
     ),
     token(TokenType.doubleEqual),
-    ColonNamedVariable(':test'),
+    ColonNamedVariable(ColonVariableToken(fakeSpan(':test'), ':test')),
   ),
   'CASE x WHEN a THEN b WHEN c THEN d ELSE e END': CaseExpression(
     base: Reference(columnName: 'x'),

sqlparser/test/parser/utils.dart

@@ -11,13 +11,11 @@ Token token(TokenType type) {
 }
 
 InlineDartToken inlineDart(String dartCode) {
-  final fakeFile = SourceFile.fromString('`$dartCode`');
-  return InlineDartToken(fakeFile.span(0));
+  return InlineDartToken(fakeSpan('`$dartCode`'));
 }
 
 IdentifierToken identifier(String content) {
-  final fakeFile = SourceFile.fromString(content);
-  return IdentifierToken(false, fakeFile.span(0));
+  return IdentifierToken(false, fakeSpan(content));
 }
 
 void testStatement(String sql, AstNode expected) {
@@ -25,6 +23,10 @@ void testStatement(String sql, AstNode expected) {
   enforceEqual(parsed, expected);
 }
 
+FileSpan fakeSpan(String content) {
+  return SourceFile.fromString(content).span(0);
+}
+
 void testAll(Map<String, AstNode> testCases) {
   testCases.forEach((sql, expected) {
     test('with $sql', () {