Scan comments in sql

moor_generator/lib/src/utils/table_reference_sorter.dart

@@ -38,11 +38,12 @@ void _visit(SpecifiedTable table, _SortRun run) {
 }
 
 class _SortRun {
-  Map<SpecifiedTable, SpecifiedTable> previous = {};
+  final Map<SpecifiedTable, SpecifiedTable> previous = {};
   final List<SpecifiedTable> result = [];
 
   /// Throws a [CircularReferenceException] because the [last] table depends on
-  /// [first], which transitively depends on [last] first.
+  /// [first], which (transitively) depends on [last] as well. The path in the
+  /// thrown exception will go from [first] to [last].
   void throwCircularException(SpecifiedTable last, SpecifiedTable first) {
     final constructedPath = <SpecifiedTable>[];
     for (var current = last; current != first; current = previous[current]) {

sqlparser/lib/src/engine/sql_engine.dart

@@ -35,6 +35,11 @@ class SqlEngine {
 
   /// Tokenizes the [source] into a list list [Token]s. Each [Token] contains
   /// information about where it appears in the [source] and a [TokenType].
+  ///
+  /// Note that the list might be tokens that should be
+  /// [Token.invisibleToParser], if you're passing them to a [Parser] directly,
+  /// you need to filter them. When using the methods in this class, this will
+  /// be taken care of automatically.
   List<Token> tokenize(String source) {
     final scanner = Scanner(source, scanMoorTokens: useMoorExtensions);
     final tokens = scanner.scanTokens();
@@ -49,7 +54,8 @@ class SqlEngine {
   /// Parses the [sql] statement into an AST-representation.
   ParseResult parse(String sql) {
     final tokens = tokenize(sql);
-    final parser = Parser(tokens, useMoor: useMoorExtensions);
+    final tokensForParser = tokens.where((t) => !t.invisibleToParser).toList();
+    final parser = Parser(tokensForParser, useMoor: useMoorExtensions);
 
     final stmt = parser.statement();
     return ParseResult._(stmt, tokens, parser.errors, sql, null);
@@ -62,7 +68,9 @@ class SqlEngine {
 
     final autoComplete = AutoCompleteEngine();
     final tokens = tokenize(content);
-    final parser = Parser(tokens, useMoor: true, autoComplete: autoComplete);
+    final tokensForParser = tokens.where((t) => !t.invisibleToParser).toList();
+    final parser =
+        Parser(tokensForParser, useMoor: true, autoComplete: autoComplete);
 
     final moorFile = parser.moorFile();
 
@@ -134,7 +142,8 @@ class ParseResult {
   /// The topmost node in the sql AST that was parsed.
   final AstNode rootNode;
 
-  /// The tokens that were scanned in the source file
+  /// The tokens that were scanned in the source file, including those that are
+  /// [Token.invisibleToParser].
   final List<Token> tokens;
 
   /// A list of all errors that occurred during parsing. [ParsingError.toString]

sqlparser/lib/src/reader/tokenizer/scanner.dart

@@ -61,13 +61,22 @@ class Scanner {
         _addToken(TokenType.plus);
         break;
       case '-':
+        if (_match('-')) {
+          _lineComment();
+        } else {
           _addToken(TokenType.minus);
+        }
         break;
       case '*':
         _addToken(TokenType.star);
         break;
       case '/':
+        if (_match('*')) {
+          _cStyleComment();
+        } else {
           _addToken(TokenType.slash);
+        }
+
         break;
       case '%':
         _addToken(TokenType.percent);
@@ -378,4 +387,37 @@ class Scanner {
       tokens.add(InlineDartToken(_currentSpan));
     }
   }
+
+  /// Scans a line comment after the -- has already been read.
+  void _lineComment() {
+    final contentBuilder = StringBuffer();
+    while (_peek() != '\n' && !_isAtEnd) {
+      contentBuilder.write(_nextChar());
+    }
+
+    tokens.add(CommentToken(
+        CommentMode.line, contentBuilder.toString(), _currentSpan));
+  }
+
+  /// Scans a /* ... */ comment after the first /* has already been read.
+  /// Note that in sqlite, these comments don't have to be terminated - they
+  /// will be closed by the end of input without causing a parsing error.
+  void _cStyleComment() {
+    final contentBuilder = StringBuffer();
+    while (!_isAtEnd) {
+      if (_match('*')) {
+        if (!_isAtEnd && _match('/')) {
+          break;
+        } else {
+          // write the * we otherwise forgot to write
+          contentBuilder.write('*');
+        }
+      } else {
+        contentBuilder.write(_nextChar());
+      }
+    }
+
+    tokens.add(CommentToken(
+        CommentMode.cStyle, contentBuilder.toString(), _currentSpan));
+  }
 }

sqlparser/lib/src/reader/tokenizer/token.dart

@@ -136,6 +136,7 @@ enum TokenType {
   action,
 
   semicolon,
+  comment,
   eof,
 
   /// Moor specific token, used to declare a type converters
@@ -252,6 +253,10 @@ const Map<String, TokenType> moorKeywords = {
 class Token {
   final TokenType type;
 
+  /// Whether this token should be invisible to the parser. We use this for
+  /// comment tokens.
+  bool get invisibleToParser => false;
+
   final FileSpan span;
   String get lexeme => span.text;
 
@@ -348,6 +353,22 @@ class KeywordToken extends Token {
   }
 }
 
+enum CommentMode { line, cStyle }
+
+/// A comment, either started with -- or with /*.
+class CommentToken extends Token {
+  final CommentMode mode;
+
+  /// The content of this comment, excluding the "--", "/*", "*/".
+  final String content;
+
+  @override
+  final bool invisibleToParser = true;
+
+  CommentToken(this.mode, this.content, FileSpan span)
+      : super(TokenType.comment, span);
+}
+
 class TokenizerError {
   final String message;
   final SourceLocation location;

sqlparser/test/parser/moor_file_test.dart

@@ -0,0 +1,67 @@
+import 'package:sqlparser/sqlparser.dart';
+import 'package:sqlparser/src/utils/ast_equality.dart';
+import 'package:test/test.dart';
+
+const content = r'''
+import 'other.dart';
+import 'another.moor';
+
+CREATE TABLE tbl (
+  id INT NOT NULL PRIMARY KEY AUTOINCREMENT,
+  -- this is a single-line comment
+  place VARCHAR REFERENCES other(location)
+)
+
+all: SELECT /* COUNT(*), */ * FROM tbl WHERE $predicate;
+''';
+
+void main() {
+  test('parses moor files', () {
+    final parsed = SqlEngine(useMoorExtensions: true).parseMoorFile(content);
+    final file = parsed.rootNode;
+
+    enforceEqual(
+      file,
+      MoorFile([
+        ImportStatement('other.dart'),
+        ImportStatement('another.moor'),
+        CreateTableStatement(
+          tableName: 'tbl',
+          columns: [
+            ColumnDefinition(
+              columnName: 'id',
+              typeName: 'INT',
+              constraints: [
+                NotNull(null),
+                PrimaryKeyColumn(null, autoIncrement: true),
+              ],
+            ),
+            ColumnDefinition(
+              columnName: 'place',
+              typeName: 'VARCHAR',
+              constraints: [
+                ForeignKeyColumnConstraint(
+                  null,
+                  ForeignKeyClause(
+                    foreignTable: TableReference('other', null),
+                    columnNames: [
+                      Reference(columnName: 'location'),
+                    ],
+                  ),
+                ),
+              ],
+            ),
+          ],
+        ),
+        DeclaredStatement(
+          'all',
+          SelectStatement(
+            columns: [StarResultColumn(null)],
+            from: [TableReference('tbl', null)],
+            where: DartExpressionPlaceholder(name: 'predicate'),
+          ),
+        ),
+      ]),
+    );
+  });
+}

sqlparser/test/scanner/comments_test.dart

@@ -0,0 +1,35 @@
+import 'package:sqlparser/src/reader/tokenizer/scanner.dart';
+import 'package:test/test.dart';
+
+import '../parser/utils.dart';
+
+void main() {
+  test('scanns comments', () {
+    const sql = r'''
+--line
+-- line
+/*c*/
+/*multi
+  line */
+/* not terminated''';
+
+    // using whereType instead of cast because of the invisible eof token
+    final tokens = Scanner(sql).scanTokens().whereType<CommentToken>();
+
+    expect(tokens.map((t) => t.mode), [
+      CommentMode.line,
+      CommentMode.line,
+      CommentMode.cStyle,
+      CommentMode.cStyle,
+      CommentMode.cStyle,
+    ]);
+
+    expect(tokens.map((t) => t.content), [
+      'line',
+      ' line',
+      'c',
+      'multi\n  line ',
+      ' not terminated',
+    ]);
+  });
+}