For custom queries, use a matching data class if possible

moor/CHANGELOG.md

@@ -1,6 +1,34 @@
 ## 1.5.0
-- More consistent and reliable migration and opening callbacks
+This version introduces some new concepts and features, which are explained in more detail below. 
-- TODO: Explain new companions
+Here is a quick overview of the new features.
+- More consistent and reliable callbacks for migrations. You can now use `MigrationStrategy.beforeOpen` 
+to run queries after migrations, but before fully opening the database. This is useful to initialize data.
+- New "update companion" classes to clearly separate between absent values and explicitly setting
+values back to null. 
+- Experimental support for compiled sql queries. Moor can now generate typesafe APIs for
+  written sql.
+
+### Update companions
+Newly introduced "Update companions" allow you to insert or update data more precisely than before.
+Previously, there was no clear separation between "null" and absent values. For instance, let's
+say we had a table "users" that stores an id, a name, and an age. Now, let's say we wanted to set
+the age of a user to null without changing its name. Would we use `User(age: null)`? Here,
+the `name` column would implicitly be set to null, so we can't cleanly separate that. However,
+with `UsersCompanion(age: Value(null))`, we know the difference between `Value(null)` and the 
+default `Value.absent()`.
+ 
+Don't worry, all your existing code will continue to work, this change is fully backwards
+compatible. You might get analyzer warnings about missing required fields. The migration to
+update companions will fix that.
+### Compiled sql queries
+Experimental support for compile time custom statements. Sounds super boring, but it
+actually gives you a fluent way to write queries in pure sql. The moor generator will figure
+out what your queries return and automatically generate the boring mapping part. Head on to
+`TODO: Documentation link` to find out how to use this new feature.
+  
+Please note that this feature is in an experimental state: Expect minor, but breaking changes
+in the API and in the generated code. Also, if you run into any issues with this feature, 
+[reporting them](https://github.com/simolus3/moor/issues/new) would be super appreciated.
 
 ## 1.4.0
 - Added the `RealColumn`, which stores floating point values

moor/lib/src/runtime/migration.dart

@@ -38,7 +38,9 @@ class MigrationStrategy {
   /// Executes after the database is ready and all migrations ran, but before
   /// any other queries will be executed, making this method suitable to
   /// populate data.
-  @Deprecated('Use beforeOpen instead')
+  @Deprecated(
+      'This callback is broken and only exists for backwards compatibility. '
+      'Use beforeOpen instead')
   final OnMigrationFinished onFinished;
 
   /// Executes after the database is ready to be used (ie. it has been opened

moor/test/data/tables/todos.g.dart

@@ -1081,8 +1081,8 @@ mixin _$SomeDaoMixin on DatabaseAccessor<TodoDb> {
   $UsersTable get users => db.users;
   $SharedTodosTable get sharedTodos => db.sharedTodos;
   $TodosTableTable get todosTable => db.todosTable;
-  TodosForUserResult _rowToTodosForUserResult(QueryRow row) {
+  TodoEntry _rowToTodoEntry(QueryRow row) {
-    return TodosForUserResult(
+    return TodoEntry(
       id: row.readInt('id'),
       title: row.readString('title'),
       content: row.readString('content'),
@@ -1091,15 +1091,15 @@ mixin _$SomeDaoMixin on DatabaseAccessor<TodoDb> {
     );
   }
 
-  Future<List<TodosForUserResult>> todosForUser(int user) {
+  Future<List<TodoEntry>> todosForUser(int user) {
     return customSelect(
         'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user',
         variables: [
           Variable.withInt(user),
-        ]).then((rows) => rows.map(_rowToTodosForUserResult).toList());
+        ]).then((rows) => rows.map(_rowToTodoEntry).toList());
   }
 
-  Stream<List<TodosForUserResult>> watchTodosForUser(int user) {
+  Stream<List<TodoEntry>> watchTodosForUser(int user) {
     return customSelectStream(
         'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user',
         variables: [
@@ -1109,21 +1109,6 @@ mixin _$SomeDaoMixin on DatabaseAccessor<TodoDb> {
           users,
           todosTable,
           sharedTodos
-        }).map((rows) => rows.map(_rowToTodosForUserResult).toList());
+        }).map((rows) => rows.map(_rowToTodoEntry).toList());
   }
 }
-
-class TodosForUserResult {
-  final int id;
-  final String title;
-  final String content;
-  final DateTime targetDate;
-  final int category;
-  TodosForUserResult({
-    this.id,
-    this.title,
-    this.content,
-    this.targetDate,
-    this.category,
-  });
-}

moor_generator/lib/src/dao_generator.dart

@@ -49,8 +49,7 @@ class DaoGenerator extends GeneratorForAnnotation<UseDao> {
     }
 
     if (queries.isNotEmpty) {
-      final parser = SqlParser(state.options, parsedTables, queries)..parse();
+      final parser = SqlParser(state, parsedTables, queries)..parse();
-      state.errors.errors.addAll(parser.errors);
 
       resolvedQueries = parser.foundQueries;
     }

moor_generator/lib/src/model/sql_query.dart

@@ -17,7 +17,12 @@ class SqlSelectQuery extends SqlQuery {
   final List<SpecifiedTable> readsFrom;
   final InferredResultSet resultSet;
 
-  String get resultClassName => '${ReCase(name).pascalCase}Result';
+  String get resultClassName {
+    if (resultSet.matchingTable != null) {
+      return resultSet.matchingTable.dartTypeName;
+    }
+    return '${ReCase(name).pascalCase}Result';
+  }
 
   SqlSelectQuery(String name, String sql, List<FoundVariable> variables,
       this.readsFrom, this.resultSet)
@@ -35,6 +40,12 @@ class InferredResultSet {
 
   InferredResultSet(this.matchingTable, this.columns);
 
+  void forceDartNames(Map<ResultColumn, String> names) {
+    _dartNames
+      ..clear()
+      ..addAll(names);
+  }
+
   /// Suggests an appropriate name that can be used as a dart field.
   String dartNameFor(ResultColumn column) {
     return _dartNames.putIfAbsent(column, () {

moor_generator/lib/src/moor_generator.dart

@@ -51,8 +51,7 @@ class MoorGenerator extends GeneratorForAnnotation<UseMoor> {
     }
 
     if (queries.isNotEmpty) {
-      final parser = SqlParser(options, tablesForThisDb, queries)..parse();
+      final parser = SqlParser(state, tablesForThisDb, queries)..parse();
-      state.errors.errors.addAll(parser.errors);
 
       resolvedQueries = parser.foundQueries;
     }

moor_generator/lib/src/parser/sql/affected_tables_visitor.dart

@@ -7,7 +7,7 @@ class AffectedTablesVisitor extends RecursiveVisitor<void> {
 
   @override
   void visitReference(Reference e) {
-    final column = e.resolved as Column;
+    final column = e.resolved;
     if (column is TableColumn) {
       foundTables.add(column.table);
     }
@@ -18,7 +18,10 @@ class AffectedTablesVisitor extends RecursiveVisitor<void> {
   @override
   void visitQueryable(Queryable e) {
     if (e is TableReference) {
-      foundTables.add(e.resolved as Table);
+      final table = e.resolved as Table;
+      if (table != null) {
+        foundTables.add(table);
+      }
     }
 
     visitChildren(e);

moor_generator/lib/src/parser/sql/query_handler.dart

@@ -0,0 +1,117 @@
+import 'package:moor_generator/src/model/sql_query.dart';
+import 'package:moor_generator/src/parser/sql/type_mapping.dart';
+import 'package:sqlparser/sqlparser.dart' hide ResultColumn;
+
+import 'affected_tables_visitor.dart';
+
+class QueryHandler {
+  final String name;
+  final AnalysisContext context;
+  final TypeMapper mapper;
+
+  Set<Table> _foundTables;
+  List<FoundVariable> _foundVariables;
+
+  SelectStatement get _select => context.root as SelectStatement;
+
+  QueryHandler(this.name, this.context, this.mapper);
+
+  SqlQuery handle() {
+    final root = context.root;
+    _foundVariables = mapper.extractVariables(context);
+
+    if (root is SelectStatement) {
+      return _handleSelect();
+    } else {
+      throw StateError(
+          'Unexpected sql: Got $root, expected a select statement');
+    }
+  }
+
+  SqlSelectQuery _handleSelect() {
+    final tableFinder = AffectedTablesVisitor();
+    _select.accept(tableFinder);
+    _foundTables = tableFinder.foundTables;
+    final moorTables = _foundTables.map(mapper.tableToMoor).toList();
+
+    return SqlSelectQuery(
+        name, context.sql, _foundVariables, moorTables, _inferResultSet());
+  }
+
+  InferredResultSet _inferResultSet() {
+    final candidatesForSingleTable = Set.of(_foundTables);
+    final columns = <ResultColumn>[];
+    final rawColumns = _select.resolvedColumns;
+
+    for (var column in rawColumns) {
+      final type = context.typeOf(column).type;
+      final moorType = mapper.resolvedToMoor(type);
+
+      columns.add(ResultColumn(column.name, moorType, type.nullable));
+
+      final table = _tableOfColumn(column);
+      candidatesForSingleTable.removeWhere((t) => t != table);
+    }
+
+    // if all columns read from the same table, and all columns in that table
+    // are present in the result set, we can use the data class we generate for
+    // that table instead of generating another class just for this result set.
+    if (candidatesForSingleTable.length == 1) {
+      final table = candidatesForSingleTable.single;
+      final moorTable = mapper.tableToMoor(table);
+
+      final resultEntryToColumn = <ResultColumn, String>{};
+      var matches = true;
+
+      // go trough all columns of the table in question
+      for (var column in moorTable.columns) {
+        // check if this column from the table is present in the result set
+        final tableColumn = table.findColumn(column.name.name);
+        final inResultSet =
+            rawColumns.where((t) => _toTableColumn(t) == tableColumn);
+
+        if (inResultSet.length == 1) {
+          // it is! Remember the correct getter name from the data class for
+          // later when we write the mapping code.
+          final columnIndex = rawColumns.indexOf(inResultSet.single);
+          resultEntryToColumn[columns[columnIndex]] = column.dartGetterName;
+        } else {
+          // it's not, so no match
+          matches = false;
+          break;
+        }
+      }
+
+      // we have established that all columns in resultEntryToColumn do appear
+      // in the moor table. Now check for set equality.
+      if (resultEntryToColumn.length != moorTable.columns.length) {
+        matches = false;
+      }
+
+      if (matches) {
+        return InferredResultSet(moorTable, columns)
+          ..forceDartNames(resultEntryToColumn);
+      }
+    }
+
+    return InferredResultSet(null, columns);
+  }
+
+  /// The table a given result column is from, or null if this column doesn't
+  /// read from a table directly.
+  Table _tableOfColumn(Column c) {
+    return _toTableColumn(c)?.table;
+  }
+
+  TableColumn _toTableColumn(Column c) {
+    if (c is TableColumn) {
+      return c;
+    } else if (c is ExpressionColumn) {
+      final expression = c.expression;
+      if (expression is Reference) {
+        return expression.resolved as TableColumn;
+      }
+    }
+    return null;
+  }
+}

moor_generator/lib/src/parser/sql/sql_parser.dart

@@ -1,87 +1,27 @@
 import 'package:analyzer/dart/constant/value.dart';
 import 'package:moor_generator/src/errors.dart';
-import 'package:moor_generator/src/model/specified_column.dart';
 import 'package:moor_generator/src/model/specified_table.dart';
 import 'package:moor_generator/src/model/sql_query.dart';
-import 'package:moor_generator/src/options.dart';
+import 'package:moor_generator/src/parser/sql/query_handler.dart';
+import 'package:moor_generator/src/parser/sql/type_mapping.dart';
+import 'package:moor_generator/src/shared_state.dart';
 import 'package:sqlparser/sqlparser.dart' hide ResultColumn;
 
-import 'affected_tables_visitor.dart';
-
 class SqlParser {
-  final MoorOptions options;
   final List<SpecifiedTable> tables;
+  final SharedState state;
   final List<DartObject> definedQueries;
 
+  final TypeMapper _mapper = TypeMapper();
   SqlEngine _engine;
-  final Map<Table, SpecifiedTable> _engineTablesToSpecified = {};
 
   final List<SqlQuery> foundQueries = [];
-  final List<MoorError> errors = [];
 
-  SqlParser(this.options, this.tables, this.definedQueries);
+  SqlParser(this.state, this.tables, this.definedQueries);
 
   void _spawnEngine() {
     _engine = SqlEngine();
-    tables.map(_extractStructure).forEach(_engine.registerTable);
+    tables.map(_mapper.extractStructure).forEach(_engine.registerTable);
-  }
-
-  /// Convert a [SpecifiedTable] from moor into something that can be understood
-  /// by the sqlparser library.
-  Table _extractStructure(SpecifiedTable table) {
-    final columns = <TableColumn>[];
-    for (var specified in table.columns) {
-      final type = _resolveForColumnType(specified.type)
-          .withNullable(specified.nullable);
-      columns.add(TableColumn(specified.name.name, type));
-    }
-
-    final engineTable = Table(name: table.sqlName, resolvedColumns: columns);
-    _engineTablesToSpecified[engineTable] = table;
-    return engineTable;
-  }
-
-  ResolvedType _resolveForColumnType(ColumnType type) {
-    switch (type) {
-      case ColumnType.integer:
-        return const ResolvedType(type: BasicType.int);
-      case ColumnType.text:
-        return const ResolvedType(type: BasicType.text);
-      case ColumnType.boolean:
-        return const ResolvedType(type: BasicType.int, hint: IsBoolean());
-      case ColumnType.datetime:
-        return const ResolvedType(type: BasicType.int, hint: IsDateTime());
-      case ColumnType.blob:
-        return const ResolvedType(type: BasicType.blob);
-      case ColumnType.real:
-        return const ResolvedType(type: BasicType.real);
-    }
-    throw StateError('cant happen');
-  }
-
-  ColumnType _resolvedToMoor(ResolvedType type) {
-    if (type == null) {
-      return ColumnType.text;
-    }
-
-    switch (type.type) {
-      case BasicType.nullType:
-        return ColumnType.text;
-      case BasicType.int:
-        if (type.hint is IsBoolean) {
-          return ColumnType.boolean;
-        } else if (type.hint is IsDateTime) {
-          return ColumnType.datetime;
-        }
-        return ColumnType.integer;
-      case BasicType.real:
-        return ColumnType.real;
-      case BasicType.text:
-        return ColumnType.text;
-      case BasicType.blob:
-        return ColumnType.blob;
-    }
-    throw StateError('Unexpected type: $type');
   }
 
   void parse() {
@@ -95,70 +35,18 @@ class SqlParser {
       try {
         context = _engine.analyze(sql);
       } catch (e, s) {
-        errors.add(MoorError(
+        state.errors.add(MoorError(
             critical: true,
             message: 'Error while trying to parse $sql: $e, $s'));
       }
 
       for (var error in context.errors) {
-        errors.add(MoorError(
+        state.errors.add(MoorError(
           message: 'The sql query $sql is invalid: ${error.message}',
         ));
       }
 
-      final root = context.root;
+      foundQueries.add(QueryHandler(name, context, _mapper).handle());
-      if (root is SelectStatement) {
-        _handleSelect(name, root, context);
-      } else {
-        throw StateError('Unexpected sql, expected a select statement');
     }
   }
-  }
-
-  void _handleSelect(
-      String queryName, SelectStatement stmt, AnalysisContext ctx) {
-    final tableFinder = AffectedTablesVisitor();
-    stmt.accept(tableFinder);
-
-    final foundTables = tableFinder.foundTables;
-    final moorTables = foundTables.map((t) => _engineTablesToSpecified[t]);
-    final resultColumns = stmt.resolvedColumns;
-
-    final moorColumns = <ResultColumn>[];
-    for (var column in resultColumns) {
-      final type = ctx.typeOf(column).type;
-      moorColumns
-          .add(ResultColumn(column.name, _resolvedToMoor(type), type.nullable));
-    }
-
-    final resultSet = InferredResultSet(null, moorColumns);
-    final foundVars = _extractVariables(ctx);
-    foundQueries.add(SqlSelectQuery(
-        queryName, ctx.sql, foundVars, moorTables.toList(), resultSet));
-  }
-
-  List<FoundVariable> _extractVariables(AnalysisContext ctx) {
-    // this contains variable references. For instance, SELECT :a = :a would
-    // contain two entries, both referring to the same variable. To do that,
-    // we use the fact that each variable has a unique index.
-    final usedVars = ctx.root.allDescendants.whereType<Variable>().toList()
-      ..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex));
-
-    final foundVariables = <FoundVariable>[];
-    var currentIndex = 0;
-
-    for (var used in usedVars) {
-      if (used.resolvedIndex == currentIndex) {
-        continue; // already handled
-      }
-
-      currentIndex++;
-      final name = (used is ColonNamedVariable) ? used.name : null;
-      final type = _resolvedToMoor(ctx.typeOf(used).type);
-
-      foundVariables.add(FoundVariable(currentIndex, name, type));
-    }
-
-    return foundVariables;
-  }
 }

moor_generator/lib/src/parser/sql/type_mapping.dart

@@ -0,0 +1,97 @@
+import 'package:moor_generator/src/model/specified_column.dart';
+import 'package:moor_generator/src/model/specified_table.dart';
+import 'package:moor_generator/src/model/sql_query.dart';
+import 'package:sqlparser/sqlparser.dart';
+
+/// Converts tables and types between the moor_generator and the sqlparser
+/// library.
+class TypeMapper {
+  final Map<Table, SpecifiedTable> _engineTablesToSpecified = {};
+
+  /// Convert a [SpecifiedTable] from moor into something that can be understood
+  /// by the sqlparser library.
+  Table extractStructure(SpecifiedTable table) {
+    final columns = <TableColumn>[];
+    for (var specified in table.columns) {
+      final type =
+          resolveForColumnType(specified.type).withNullable(specified.nullable);
+      columns.add(TableColumn(specified.name.name, type));
+    }
+
+    final engineTable = Table(name: table.sqlName, resolvedColumns: columns);
+    _engineTablesToSpecified[engineTable] = table;
+    return engineTable;
+  }
+
+  ResolvedType resolveForColumnType(ColumnType type) {
+    switch (type) {
+      case ColumnType.integer:
+        return const ResolvedType(type: BasicType.int);
+      case ColumnType.text:
+        return const ResolvedType(type: BasicType.text);
+      case ColumnType.boolean:
+        return const ResolvedType(type: BasicType.int, hint: IsBoolean());
+      case ColumnType.datetime:
+        return const ResolvedType(type: BasicType.int, hint: IsDateTime());
+      case ColumnType.blob:
+        return const ResolvedType(type: BasicType.blob);
+      case ColumnType.real:
+        return const ResolvedType(type: BasicType.real);
+    }
+    throw StateError('cant happen');
+  }
+
+  ColumnType resolvedToMoor(ResolvedType type) {
+    if (type == null) {
+      return ColumnType.text;
+    }
+
+    switch (type.type) {
+      case BasicType.nullType:
+        return ColumnType.text;
+      case BasicType.int:
+        if (type.hint is IsBoolean) {
+          return ColumnType.boolean;
+        } else if (type.hint is IsDateTime) {
+          return ColumnType.datetime;
+        }
+        return ColumnType.integer;
+      case BasicType.real:
+        return ColumnType.real;
+      case BasicType.text:
+        return ColumnType.text;
+      case BasicType.blob:
+        return ColumnType.blob;
+    }
+    throw StateError('Unexpected type: $type');
+  }
+
+  List<FoundVariable> extractVariables(AnalysisContext ctx) {
+    // this contains variable references. For instance, SELECT :a = :a would
+    // contain two entries, both referring to the same variable. To do that,
+    // we use the fact that each variable has a unique index.
+    final usedVars = ctx.root.allDescendants.whereType<Variable>().toList()
+      ..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex));
+
+    final foundVariables = <FoundVariable>[];
+    var currentIndex = 0;
+
+    for (var used in usedVars) {
+      if (used.resolvedIndex == currentIndex) {
+        continue; // already handled
+      }
+
+      currentIndex++;
+      final name = (used is ColonNamedVariable) ? used.name : null;
+      final type = resolvedToMoor(ctx.typeOf(used).type);
+
+      foundVariables.add(FoundVariable(currentIndex, name, type));
+    }
+
+    return foundVariables;
+  }
+
+  SpecifiedTable tableToMoor(Table table) {
+    return _engineTablesToSpecified[table];
+  }
+}

sqlparser/README.md

@@ -1,7 +1,7 @@
 # sqlparser
 
-An sql parser and static analyzer, written in pure Dart. At the moment, only `SELECT` statements
+An sql parser and static analyzer, written in pure Dart. At the moment, only `SELECT` and `DELETE`
-are supported.
+statements are supported. Further, this library only targets the sqlite dialect at the time being.
 
 ## Features
 Not all features are available yet, put parsing select statements (even complex ones!) and
@@ -56,6 +56,10 @@ resolvedColumns.map((c) => c.name)); // id, content, id, content, 3 + 4
 resolvedColumns.map((c) => context.typeOf(c).type.type) // int, text, int, text, int, int
 ```
 
+## But why?
+[Moor](https://pub.dev/packages/moor_flutter), a persistence library for Flutter apps, uses this
+package to generate type-safe methods from sql.
+
 ## Limitations
 - For now, only `SELECT` and `DELETE` expressions are implemented, `UPDATE` and `INSERT` will follow
   soon.

sqlparser/pubspec.yaml

@@ -10,6 +10,7 @@ environment:
   sdk: '>=2.2.2 <3.0.0'
   meta: ^1.1.7
   collection: ^1.14.11
+  source_span: ^1.5.5
 
 dev_dependencies:
   test: ^1.0.0