Parse specified tables from .moor files

moor_generator/lib/src/model/specified_table.dart

@@ -6,9 +6,16 @@ import 'package:recase/recase.dart';
 /// A parsed table, declared in code by extending `Table` and referencing that
 /// table in `@UseMoor` or `@UseDao`.
 class SpecifiedTable {
-  /// The [ClassElement] for the class that declares this table.
+  /// The [ClassElement] for the class that declares this table or null if
+  /// the table was inferred from a `CREATE TABLE` statement.
   final ClassElement fromClass;
 
+  /// If [fromClass] is null, another source to use when determining the name
+  /// of this table in generated Dart code.
+  final String _overriddenName;
+
+  String get _baseName => _overriddenName ?? fromClass.name;
+
   /// The columns declared in this table.
   final List<SpecifiedColumn> columns;
 
@@ -18,9 +25,9 @@ class SpecifiedTable {
   /// The name for the data class associated with this table
   final String dartTypeName;
 
-  String get tableFieldName => ReCase(fromClass.name).camelCase;
-  String get tableInfoName => tableInfoNameForTableClass(fromClass);
-  String get updateCompanionName => _updateCompanionName(fromClass);
+  String get tableFieldName => ReCase(_baseName).camelCase;
+  String get tableInfoName => tableInfoNameForTableClass(_baseName);
+  String get updateCompanionName => _updateCompanionName(_baseName);
 
   /// The set of primary keys, if they have been explicitly defined by
   /// overriding `primaryKey` in the table class. `null` if the primary key has
@@ -32,15 +39,15 @@ class SpecifiedTable {
       this.columns,
       this.sqlName,
       this.dartTypeName,
-      this.primaryKey});
+      this.primaryKey,
+      String overriddenName})
+      : _overriddenName = overriddenName;
 
   /// Finds all type converters used in this tables.
   Iterable<UsedTypeConverter> get converters =>
       columns.map((c) => c.typeConverter).where((t) => t != null);
 }
 
-String tableInfoNameForTableClass(ClassElement fromClass) =>
-    '\$${fromClass.name}Table';
+String tableInfoNameForTableClass(String className) => '\$${className}Table';
 
-String _updateCompanionName(ClassElement fromClass) =>
-    '${fromClass.name}Companion';
+String _updateCompanionName(String className) => '${className}Companion';

moor_generator/lib/src/parser/moor/moor_analyzer.dart

@@ -0,0 +1,63 @@
+import 'package:moor_generator/src/parser/moor/parsed_moor_file.dart';
+import 'package:source_span/source_span.dart';
+import 'package:sqlparser/sqlparser.dart';
+
+/// Parses and analyzes the experimental `.moor` files containing sql
+/// statements.
+class MoorAnalyzer {
+  /// Content of the `.moor` file we're analyzing.
+  final String content;
+
+  MoorAnalyzer(this.content);
+
+  Future<MoorParsingResult> analyze() {
+    final results = SqlEngine().parseMultiple(content);
+
+    final createdTables = <CreateTable>[];
+    final errors = <MoorParsingError>[];
+
+    for (var parsedStmt in results) {
+      if (parsedStmt.rootNode is CreateTableStatement) {
+        createdTables.add(CreateTable(parsedStmt));
+      } else {
+        errors.add(
+          MoorParsingError(
+            parsedStmt.rootNode.span,
+            message:
+                'At the moment, only CREATE TABLE statements are supported in .moor files',
+          ),
+        );
+      }
+    }
+
+    // all results have the same list of errors
+    final sqlErrors = results.isEmpty ? <ParsingError>[] : results.first.errors;
+
+    for (var error in sqlErrors) {
+      errors.add(MoorParsingError(error.token.span, message: error.message));
+    }
+
+    final parsedFile = ParsedMoorFile(createdTables);
+
+    return Future.value(MoorParsingResult(parsedFile, errors));
+  }
+}
+
+class MoorParsingResult {
+  final ParsedMoorFile parsedFile;
+  final List<MoorParsingError> errors;
+
+  MoorParsingResult(this.parsedFile, this.errors);
+}
+
+class MoorParsingError {
+  final FileSpan span;
+  final String message;
+
+  MoorParsingError(this.span, {this.message});
+
+  @override
+  String toString() {
+    return span.message(message, color: true);
+  }
+}

moor_generator/lib/src/parser/moor/parsed_moor_file.dart

@@ -0,0 +1,94 @@
+import 'package:moor_generator/src/model/specified_column.dart';
+import 'package:moor_generator/src/model/specified_table.dart';
+import 'package:moor_generator/src/parser/sql/type_mapping.dart';
+import 'package:moor_generator/src/utils/names.dart';
+import 'package:recase/recase.dart';
+import 'package:sqlparser/sqlparser.dart';
+
+/*
+We're in the process of defining what a .moor file could actually look like.
+At the moment, we only support "CREATE TABLE" statements:
+``` // content of a .moor file
+CREATE TABLE users (
+  id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT,
+  name VARCHAR(100) NOT NULL,
+)
+```
+
+In the future, we'd also like to support
+- import statements between moor files
+- import statements from moor files referencing tables declared via the Dart DSL
+- declaring statements in these files, similar to how compiled statements work
+  with the annotation.
+ */
+
+class ParsedMoorFile {
+  final List<CreateTable> declaredTables;
+
+  ParsedMoorFile(this.declaredTables);
+}
+
+class CreateTable {
+  /// The AST of this `CREATE TABLE` statement.
+  final ParseResult ast;
+
+  SpecifiedTable extractTable(TypeMapper mapper) {
+    final table =
+        SchemaFromCreateTable().read(ast.rootNode as CreateTableStatement);
+
+    final foundColumns = <String, SpecifiedColumn>{};
+    final primaryKey = <SpecifiedColumn>{};
+
+    for (var column in table.resolvedColumns) {
+      var isPrimaryKey = false;
+      final features = <ColumnFeature>[];
+      final sqlName = column.name;
+      final dartName = ReCase(sqlName).camelCase;
+      final constraintWriter = StringBuffer();
+
+      for (var constraint in column.constraints) {
+        if (constraint is PrimaryKeyColumn) {
+          isPrimaryKey = true;
+          if (constraint.autoIncrement) {
+            features.add(AutoIncrement());
+          }
+        }
+
+        if (constraintWriter.isNotEmpty) {
+          constraintWriter.write(' ');
+        }
+        constraintWriter.write(constraint.span.text);
+      }
+
+      final parsed = SpecifiedColumn(
+        type: mapper.resolvedToMoor(column.type),
+        nullable: column.type.nullable,
+        dartGetterName: dartName,
+        name: ColumnName.implicitly(sqlName),
+        declaredAsPrimaryKey: isPrimaryKey,
+        features: features,
+        customConstraints: constraintWriter.toString(),
+      );
+
+      foundColumns[column.name] = parsed;
+      if (isPrimaryKey) {
+        primaryKey.add(parsed);
+      }
+    }
+
+    final tableName = table.name;
+    final dartTableName = ReCase(tableName).pascalCase;
+
+    // todo respect WITHOUT ROWID clause and table constraints
+    return SpecifiedTable(
+      fromClass: null,
+      columns: foundColumns.values.toList(),
+      sqlName: table.name,
+      dartTypeName: dataClassNameForClassName(dartTableName),
+      overriddenName: dartTableName,
+      primaryKey: primaryKey,
+    );
+  }
+
+  CreateTable(this.ast);
+}

moor_generator/lib/src/writer/table_writer.dart

@@ -24,7 +24,7 @@ class TableWriter {
 
   void writeTableInfoClass(StringBuffer buffer) {
     final dataClass = table.dartTypeName;
-    final tableDslName = table.fromClass.name;
+    final tableDslName = table.fromClass?.name ?? 'dynamic';
 
     // class UsersTable extends Users implements TableInfo<Users, User> {
     buffer

moor_generator/test/parser/moor/moor_parser_test.dart

@@ -0,0 +1,24 @@
+import 'package:moor_generator/src/parser/moor/moor_analyzer.dart';
+import 'package:moor_generator/src/parser/sql/type_mapping.dart';
+import 'package:test_api/test_api.dart';
+
+void main() {
+  final content = '''
+CREATE TABLE users(
+  id INT NOT NULL PRIMARY KEY AUTOINCREMENT,
+  name VARCHAR NOT NULL CHECK(LENGTH(name) BETWEEN 5 AND 30)
+);
+  ''';
+
+  test('extracts table structure from .moor files', () async {
+    final analyzer = MoorAnalyzer(content);
+    final result = await analyzer.analyze();
+
+    expect(result.errors, isEmpty);
+
+    final table =
+        result.parsedFile.declaredTables.single.extractTable(TypeMapper());
+
+    expect(table.sqlName, 'users');
+  });
+}

sqlparser/lib/sqlparser.dart

@@ -4,4 +4,5 @@ library sqlparser;
 export 'src/analysis/analysis.dart';
 export 'src/ast/ast.dart';
 export 'src/engine/sql_engine.dart';
+export 'src/reader/parser/parser.dart' show ParsingError;
 export 'src/reader/tokenizer/token.dart' show CumulatedTokenizerException;

sqlparser/lib/src/ast/ast.dart

@@ -1,5 +1,6 @@
 import 'package:collection/collection.dart';
 import 'package:meta/meta.dart';
+import 'package:source_span/source_span.dart';
 import 'package:sqlparser/src/reader/tokenizer/token.dart';
 import 'package:sqlparser/src/analysis/analysis.dart';
 
@@ -49,6 +50,8 @@ abstract class AstNode {
   /// The last position that belongs to node, exclusive. Not set for all nodes.
   int get lastPosition => last.span.end.offset;
 
+  FileSpan get span => first.span.expand(last.span);
+
   /// Sets the [AstNode.first] and [AstNode.last] property in one go.
   void setSpan(Token first, Token last) {
     this.first = first;

sqlparser/lib/src/engine/sql_engine.dart

@@ -25,34 +25,68 @@ class SqlEngine {
     return scope;
   }
 
-  /// Parses the [sql] statement. At the moment, only SELECT statements are
-  /// supported.
-  ParseResult parse(String sql) {
-    final scanner = Scanner(sql);
+  /// Tokenizes the [source] into a list list [Token]s. Each [Token] contains
+  /// information about where it appears in the [source] and a [TokenType].
+  List<Token> tokenize(String source) {
+    final scanner = Scanner(source);
     final tokens = scanner.scanTokens();
 
     if (scanner.errors.isNotEmpty) {
       throw CumulatedTokenizerException(scanner.errors);
     }
 
-    final parser = Parser(tokens);
-    final stmt = parser.statement();
-    return ParseResult._(stmt, parser.errors);
+    return tokens;
   }
 
-  /// Parses and analyzes the [sql] statement, which at the moment has to be a
-  /// select statement. The [AnalysisContext] returned contains all information
-  /// about type hints, errors, and the parsed AST.
+  /// Parses the [sql] statement into an AST-representation.
+  ParseResult parse(String sql) {
+    final tokens = tokenize(sql);
+    final parser = Parser(tokens);
+
+    final stmt = parser.statement();
+    return ParseResult._(stmt, parser.errors, sql);
+  }
+
+  /// Parses multiple sql statements, separated by a semicolon. All
+  /// [ParseResult] entries will have the same [ParseResult.errors], but the
+  /// [ParseResult.sql] will only refer to the substring creating a statement.
+  List<ParseResult> parseMultiple(String sql) {
+    final tokens = tokenize(sql);
+    final parser = Parser(tokens);
+
+    final stmts = parser.statements();
+
+    return stmts.map((statement) {
+      final first = statement.firstPosition;
+      final last = statement.lastPosition;
+
+      final source = sql.substring(first, last);
+      return ParseResult._(statement, parser.errors, source);
+    }).toList();
+  }
+
+  /// Parses and analyzes the [sql] statement. The [AnalysisContext] returned
+  /// contains all information about type hints, errors, and the parsed AST.
   ///
   /// The analyzer needs to know all the available tables to resolve references
   /// and result columns, so all known tables should be registered using
   /// [registerTable] before calling this method.
   AnalysisContext analyze(String sql) {
     final result = parse(sql);
+    return analyzeParsed(result);
+  }
+
+  /// Analyzes a parsed [result] statement. The [AnalysisContext] returned
+  /// contains all information about type hints, errors, and the parsed AST.
+  ///
+  /// The analyzer needs to know all the available tables to resolve references
+  /// and result columns, so all known tables should be registered using
+  /// [registerTable] before calling this method.
+  AnalysisContext analyzeParsed(ParseResult result) {
     final node = result.rootNode;
     const SetParentVisitor().startAtRoot(node);
 
-    final context = AnalysisContext(node, sql);
+    final context = AnalysisContext(node, result.sql);
     final scope = _constructRootScope();
 
     try {
@@ -84,5 +118,8 @@ class ParseResult {
   /// where the error occurred.
   final List<ParsingError> errors;
 
-  ParseResult._(this.rootNode, this.errors);
+  /// The sql source that created the AST at [rootNode].
+  final String sql;
+
+  ParseResult._(this.rootNode, this.errors, this.sql);
 }

sqlparser/lib/src/reader/parser/parser.dart

@@ -134,13 +134,26 @@ class Parser extends ParserBase
     with ExpressionParser, SchemaParser, CrudParser {
   Parser(List<Token> tokens) : super(tokens);
 
-  Statement statement() {
+  Statement statement({bool expectEnd = true}) {
+    final first = _peek;
     final stmt = select() ?? _deleteStmt() ?? _update() ?? _createTable();
 
+    if (stmt == null) {
+      _error('Expected a sql statement to start here');
+    }
+
     _matchOne(TokenType.semicolon);
-    if (!_isAtEnd) {
+    if (!_isAtEnd && expectEnd) {
       _error('Expected the statement to finish here');
     }
-    return stmt;
+    return stmt..setSpan(first, _previous);
+  }
+
+  List<Statement> statements() {
+    final stmts = <Statement>[];
+    while (!_isAtEnd) {
+      stmts.add(statement(expectEnd: false));
+    }
+    return stmts;
   }
 }