diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 52712213..d2934068 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,6 +1,34 @@ ## 1.5.0 -- More consistent and reliable migration and opening callbacks -- TODO: Explain new companions +This version introduces some new concepts and features, which are explained in more detail below. +Here is a quick overview of the new features. +- More consistent and reliable callbacks for migrations. You can now use `MigrationStrategy.beforeOpen` +to run queries after migrations, but before fully opening the database. This is useful to initialize data. +- New "update companion" classes to clearly separate between absent values and explicitly setting +values back to null. +- Experimental support for compiled sql queries. Moor can now generate typesafe APIs for + written sql. + +### Update companions +Newly introduced "Update companions" allow you to insert or update data more precisely than before. +Previously, there was no clear separation between "null" and absent values. For instance, let's +say we had a table "users" that stores an id, a name, and an age. Now, let's say we wanted to set +the age of a user to null without changing its name. Would we use `User(age: null)`? Here, +the `name` column would implicitly be set to null, so we can't cleanly separate that. However, +with `UsersCompanion(age: Value(null))`, we know the difference between `Value(null)` and the +default `Value.absent()`. + +Don't worry, all your existing code will continue to work, this change is fully backwards +compatible. You might get analyzer warnings about missing required fields. The migration to +update companions will fix that. +### Compiled sql queries +Experimental support for compile time custom statements. Sounds super boring, but it +actually gives you a fluent way to write queries in pure sql. The moor generator will figure +out what your queries return and automatically generate the boring mapping part. Head on to +`TODO: Documentation link` to find out how to use this new feature. + +Please note that this feature is in an experimental state: Expect minor, but breaking changes +in the API and in the generated code. Also, if you run into any issues with this feature, +[reporting them](https://github.com/simolus3/moor/issues/new) would be super appreciated. ## 1.4.0 - Added the `RealColumn`, which stores floating point values diff --git a/moor/lib/src/runtime/migration.dart b/moor/lib/src/runtime/migration.dart index 72e7ca51..ed428ff9 100644 --- a/moor/lib/src/runtime/migration.dart +++ b/moor/lib/src/runtime/migration.dart @@ -38,7 +38,9 @@ class MigrationStrategy { /// Executes after the database is ready and all migrations ran, but before /// any other queries will be executed, making this method suitable to /// populate data. - @Deprecated('Use beforeOpen instead') + @Deprecated( + 'This callback is broken and only exists for backwards compatibility. ' + 'Use beforeOpen instead') final OnMigrationFinished onFinished; /// Executes after the database is ready to be used (ie. it has been opened diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 24909d5c..a5710d60 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1081,8 +1081,8 @@ mixin _$SomeDaoMixin on DatabaseAccessor { $UsersTable get users => db.users; $SharedTodosTable get sharedTodos => db.sharedTodos; $TodosTableTable get todosTable => db.todosTable; - TodosForUserResult _rowToTodosForUserResult(QueryRow row) { - return TodosForUserResult( + TodoEntry _rowToTodoEntry(QueryRow row) { + return TodoEntry( id: row.readInt('id'), title: row.readString('title'), content: row.readString('content'), @@ -1091,15 +1091,15 @@ mixin _$SomeDaoMixin on DatabaseAccessor { ); } - Future> todosForUser(int user) { + Future> todosForUser(int user) { return customSelect( 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', variables: [ Variable.withInt(user), - ]).then((rows) => rows.map(_rowToTodosForUserResult).toList()); + ]).then((rows) => rows.map(_rowToTodoEntry).toList()); } - Stream> watchTodosForUser(int user) { + Stream> watchTodosForUser(int user) { return customSelectStream( 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', variables: [ @@ -1109,21 +1109,6 @@ mixin _$SomeDaoMixin on DatabaseAccessor { users, todosTable, sharedTodos - }).map((rows) => rows.map(_rowToTodosForUserResult).toList()); + }).map((rows) => rows.map(_rowToTodoEntry).toList()); } } - -class TodosForUserResult { - final int id; - final String title; - final String content; - final DateTime targetDate; - final int category; - TodosForUserResult({ - this.id, - this.title, - this.content, - this.targetDate, - this.category, - }); -} diff --git a/moor_generator/lib/src/dao_generator.dart b/moor_generator/lib/src/dao_generator.dart index 772dab0f..313faece 100644 --- a/moor_generator/lib/src/dao_generator.dart +++ b/moor_generator/lib/src/dao_generator.dart @@ -49,8 +49,7 @@ class DaoGenerator extends GeneratorForAnnotation { } if (queries.isNotEmpty) { - final parser = SqlParser(state.options, parsedTables, queries)..parse(); - state.errors.errors.addAll(parser.errors); + final parser = SqlParser(state, parsedTables, queries)..parse(); resolvedQueries = parser.foundQueries; } diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 8703d4d1..36396d28 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -17,7 +17,12 @@ class SqlSelectQuery extends SqlQuery { final List readsFrom; final InferredResultSet resultSet; - String get resultClassName => '${ReCase(name).pascalCase}Result'; + String get resultClassName { + if (resultSet.matchingTable != null) { + return resultSet.matchingTable.dartTypeName; + } + return '${ReCase(name).pascalCase}Result'; + } SqlSelectQuery(String name, String sql, List variables, this.readsFrom, this.resultSet) @@ -35,6 +40,12 @@ class InferredResultSet { InferredResultSet(this.matchingTable, this.columns); + void forceDartNames(Map names) { + _dartNames + ..clear() + ..addAll(names); + } + /// Suggests an appropriate name that can be used as a dart field. String dartNameFor(ResultColumn column) { return _dartNames.putIfAbsent(column, () { diff --git a/moor_generator/lib/src/moor_generator.dart b/moor_generator/lib/src/moor_generator.dart index f1210ae7..f8807c48 100644 --- a/moor_generator/lib/src/moor_generator.dart +++ b/moor_generator/lib/src/moor_generator.dart @@ -51,8 +51,7 @@ class MoorGenerator extends GeneratorForAnnotation { } if (queries.isNotEmpty) { - final parser = SqlParser(options, tablesForThisDb, queries)..parse(); - state.errors.errors.addAll(parser.errors); + final parser = SqlParser(state, tablesForThisDb, queries)..parse(); resolvedQueries = parser.foundQueries; } diff --git a/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart b/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart index 0cc2d7ea..85700bce 100644 --- a/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart +++ b/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart @@ -7,7 +7,7 @@ class AffectedTablesVisitor extends RecursiveVisitor { @override void visitReference(Reference e) { - final column = e.resolved as Column; + final column = e.resolved; if (column is TableColumn) { foundTables.add(column.table); } @@ -18,7 +18,10 @@ class AffectedTablesVisitor extends RecursiveVisitor { @override void visitQueryable(Queryable e) { if (e is TableReference) { - foundTables.add(e.resolved as Table); + final table = e.resolved as Table; + if (table != null) { + foundTables.add(table); + } } visitChildren(e); diff --git a/moor_generator/lib/src/parser/sql/query_handler.dart b/moor_generator/lib/src/parser/sql/query_handler.dart new file mode 100644 index 00000000..6d4a1a11 --- /dev/null +++ b/moor_generator/lib/src/parser/sql/query_handler.dart @@ -0,0 +1,117 @@ +import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:moor_generator/src/parser/sql/type_mapping.dart'; +import 'package:sqlparser/sqlparser.dart' hide ResultColumn; + +import 'affected_tables_visitor.dart'; + +class QueryHandler { + final String name; + final AnalysisContext context; + final TypeMapper mapper; + + Set _foundTables; + List _foundVariables; + + SelectStatement get _select => context.root as SelectStatement; + + QueryHandler(this.name, this.context, this.mapper); + + SqlQuery handle() { + final root = context.root; + _foundVariables = mapper.extractVariables(context); + + if (root is SelectStatement) { + return _handleSelect(); + } else { + throw StateError( + 'Unexpected sql: Got $root, expected a select statement'); + } + } + + SqlSelectQuery _handleSelect() { + final tableFinder = AffectedTablesVisitor(); + _select.accept(tableFinder); + _foundTables = tableFinder.foundTables; + final moorTables = _foundTables.map(mapper.tableToMoor).toList(); + + return SqlSelectQuery( + name, context.sql, _foundVariables, moorTables, _inferResultSet()); + } + + InferredResultSet _inferResultSet() { + final candidatesForSingleTable = Set.of(_foundTables); + final columns = []; + final rawColumns = _select.resolvedColumns; + + for (var column in rawColumns) { + final type = context.typeOf(column).type; + final moorType = mapper.resolvedToMoor(type); + + columns.add(ResultColumn(column.name, moorType, type.nullable)); + + final table = _tableOfColumn(column); + candidatesForSingleTable.removeWhere((t) => t != table); + } + + // if all columns read from the same table, and all columns in that table + // are present in the result set, we can use the data class we generate for + // that table instead of generating another class just for this result set. + if (candidatesForSingleTable.length == 1) { + final table = candidatesForSingleTable.single; + final moorTable = mapper.tableToMoor(table); + + final resultEntryToColumn = {}; + var matches = true; + + // go trough all columns of the table in question + for (var column in moorTable.columns) { + // check if this column from the table is present in the result set + final tableColumn = table.findColumn(column.name.name); + final inResultSet = + rawColumns.where((t) => _toTableColumn(t) == tableColumn); + + if (inResultSet.length == 1) { + // it is! Remember the correct getter name from the data class for + // later when we write the mapping code. + final columnIndex = rawColumns.indexOf(inResultSet.single); + resultEntryToColumn[columns[columnIndex]] = column.dartGetterName; + } else { + // it's not, so no match + matches = false; + break; + } + } + + // we have established that all columns in resultEntryToColumn do appear + // in the moor table. Now check for set equality. + if (resultEntryToColumn.length != moorTable.columns.length) { + matches = false; + } + + if (matches) { + return InferredResultSet(moorTable, columns) + ..forceDartNames(resultEntryToColumn); + } + } + + return InferredResultSet(null, columns); + } + + /// The table a given result column is from, or null if this column doesn't + /// read from a table directly. + Table _tableOfColumn(Column c) { + return _toTableColumn(c)?.table; + } + + TableColumn _toTableColumn(Column c) { + if (c is TableColumn) { + return c; + } else if (c is ExpressionColumn) { + final expression = c.expression; + if (expression is Reference) { + return expression.resolved as TableColumn; + } + } + return null; + } +} diff --git a/moor_generator/lib/src/parser/sql/sql_parser.dart b/moor_generator/lib/src/parser/sql/sql_parser.dart index bc8fae23..083c0c21 100644 --- a/moor_generator/lib/src/parser/sql/sql_parser.dart +++ b/moor_generator/lib/src/parser/sql/sql_parser.dart @@ -1,87 +1,27 @@ import 'package:analyzer/dart/constant/value.dart'; import 'package:moor_generator/src/errors.dart'; -import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; -import 'package:moor_generator/src/options.dart'; +import 'package:moor_generator/src/parser/sql/query_handler.dart'; +import 'package:moor_generator/src/parser/sql/type_mapping.dart'; +import 'package:moor_generator/src/shared_state.dart'; import 'package:sqlparser/sqlparser.dart' hide ResultColumn; -import 'affected_tables_visitor.dart'; - class SqlParser { - final MoorOptions options; final List tables; + final SharedState state; final List definedQueries; + final TypeMapper _mapper = TypeMapper(); SqlEngine _engine; - final Map _engineTablesToSpecified = {}; final List foundQueries = []; - final List errors = []; - SqlParser(this.options, this.tables, this.definedQueries); + SqlParser(this.state, this.tables, this.definedQueries); void _spawnEngine() { _engine = SqlEngine(); - tables.map(_extractStructure).forEach(_engine.registerTable); - } - - /// Convert a [SpecifiedTable] from moor into something that can be understood - /// by the sqlparser library. - Table _extractStructure(SpecifiedTable table) { - final columns = []; - for (var specified in table.columns) { - final type = _resolveForColumnType(specified.type) - .withNullable(specified.nullable); - columns.add(TableColumn(specified.name.name, type)); - } - - final engineTable = Table(name: table.sqlName, resolvedColumns: columns); - _engineTablesToSpecified[engineTable] = table; - return engineTable; - } - - ResolvedType _resolveForColumnType(ColumnType type) { - switch (type) { - case ColumnType.integer: - return const ResolvedType(type: BasicType.int); - case ColumnType.text: - return const ResolvedType(type: BasicType.text); - case ColumnType.boolean: - return const ResolvedType(type: BasicType.int, hint: IsBoolean()); - case ColumnType.datetime: - return const ResolvedType(type: BasicType.int, hint: IsDateTime()); - case ColumnType.blob: - return const ResolvedType(type: BasicType.blob); - case ColumnType.real: - return const ResolvedType(type: BasicType.real); - } - throw StateError('cant happen'); - } - - ColumnType _resolvedToMoor(ResolvedType type) { - if (type == null) { - return ColumnType.text; - } - - switch (type.type) { - case BasicType.nullType: - return ColumnType.text; - case BasicType.int: - if (type.hint is IsBoolean) { - return ColumnType.boolean; - } else if (type.hint is IsDateTime) { - return ColumnType.datetime; - } - return ColumnType.integer; - case BasicType.real: - return ColumnType.real; - case BasicType.text: - return ColumnType.text; - case BasicType.blob: - return ColumnType.blob; - } - throw StateError('Unexpected type: $type'); + tables.map(_mapper.extractStructure).forEach(_engine.registerTable); } void parse() { @@ -95,70 +35,18 @@ class SqlParser { try { context = _engine.analyze(sql); } catch (e, s) { - errors.add(MoorError( + state.errors.add(MoorError( critical: true, message: 'Error while trying to parse $sql: $e, $s')); } for (var error in context.errors) { - errors.add(MoorError( + state.errors.add(MoorError( message: 'The sql query $sql is invalid: ${error.message}', )); } - final root = context.root; - if (root is SelectStatement) { - _handleSelect(name, root, context); - } else { - throw StateError('Unexpected sql, expected a select statement'); - } + foundQueries.add(QueryHandler(name, context, _mapper).handle()); } } - - void _handleSelect( - String queryName, SelectStatement stmt, AnalysisContext ctx) { - final tableFinder = AffectedTablesVisitor(); - stmt.accept(tableFinder); - - final foundTables = tableFinder.foundTables; - final moorTables = foundTables.map((t) => _engineTablesToSpecified[t]); - final resultColumns = stmt.resolvedColumns; - - final moorColumns = []; - for (var column in resultColumns) { - final type = ctx.typeOf(column).type; - moorColumns - .add(ResultColumn(column.name, _resolvedToMoor(type), type.nullable)); - } - - final resultSet = InferredResultSet(null, moorColumns); - final foundVars = _extractVariables(ctx); - foundQueries.add(SqlSelectQuery( - queryName, ctx.sql, foundVars, moorTables.toList(), resultSet)); - } - - List _extractVariables(AnalysisContext ctx) { - // this contains variable references. For instance, SELECT :a = :a would - // contain two entries, both referring to the same variable. To do that, - // we use the fact that each variable has a unique index. - final usedVars = ctx.root.allDescendants.whereType().toList() - ..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex)); - - final foundVariables = []; - var currentIndex = 0; - - for (var used in usedVars) { - if (used.resolvedIndex == currentIndex) { - continue; // already handled - } - - currentIndex++; - final name = (used is ColonNamedVariable) ? used.name : null; - final type = _resolvedToMoor(ctx.typeOf(used).type); - - foundVariables.add(FoundVariable(currentIndex, name, type)); - } - - return foundVariables; - } } diff --git a/moor_generator/lib/src/parser/sql/type_mapping.dart b/moor_generator/lib/src/parser/sql/type_mapping.dart new file mode 100644 index 00000000..c802948b --- /dev/null +++ b/moor_generator/lib/src/parser/sql/type_mapping.dart @@ -0,0 +1,97 @@ +import 'package:moor_generator/src/model/specified_column.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:sqlparser/sqlparser.dart'; + +/// Converts tables and types between the moor_generator and the sqlparser +/// library. +class TypeMapper { + final Map _engineTablesToSpecified = {}; + + /// Convert a [SpecifiedTable] from moor into something that can be understood + /// by the sqlparser library. + Table extractStructure(SpecifiedTable table) { + final columns = []; + for (var specified in table.columns) { + final type = + resolveForColumnType(specified.type).withNullable(specified.nullable); + columns.add(TableColumn(specified.name.name, type)); + } + + final engineTable = Table(name: table.sqlName, resolvedColumns: columns); + _engineTablesToSpecified[engineTable] = table; + return engineTable; + } + + ResolvedType resolveForColumnType(ColumnType type) { + switch (type) { + case ColumnType.integer: + return const ResolvedType(type: BasicType.int); + case ColumnType.text: + return const ResolvedType(type: BasicType.text); + case ColumnType.boolean: + return const ResolvedType(type: BasicType.int, hint: IsBoolean()); + case ColumnType.datetime: + return const ResolvedType(type: BasicType.int, hint: IsDateTime()); + case ColumnType.blob: + return const ResolvedType(type: BasicType.blob); + case ColumnType.real: + return const ResolvedType(type: BasicType.real); + } + throw StateError('cant happen'); + } + + ColumnType resolvedToMoor(ResolvedType type) { + if (type == null) { + return ColumnType.text; + } + + switch (type.type) { + case BasicType.nullType: + return ColumnType.text; + case BasicType.int: + if (type.hint is IsBoolean) { + return ColumnType.boolean; + } else if (type.hint is IsDateTime) { + return ColumnType.datetime; + } + return ColumnType.integer; + case BasicType.real: + return ColumnType.real; + case BasicType.text: + return ColumnType.text; + case BasicType.blob: + return ColumnType.blob; + } + throw StateError('Unexpected type: $type'); + } + + List extractVariables(AnalysisContext ctx) { + // this contains variable references. For instance, SELECT :a = :a would + // contain two entries, both referring to the same variable. To do that, + // we use the fact that each variable has a unique index. + final usedVars = ctx.root.allDescendants.whereType().toList() + ..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex)); + + final foundVariables = []; + var currentIndex = 0; + + for (var used in usedVars) { + if (used.resolvedIndex == currentIndex) { + continue; // already handled + } + + currentIndex++; + final name = (used is ColonNamedVariable) ? used.name : null; + final type = resolvedToMoor(ctx.typeOf(used).type); + + foundVariables.add(FoundVariable(currentIndex, name, type)); + } + + return foundVariables; + } + + SpecifiedTable tableToMoor(Table table) { + return _engineTablesToSpecified[table]; + } +} diff --git a/sqlparser/README.md b/sqlparser/README.md index 21c3942f..dc80157c 100644 --- a/sqlparser/README.md +++ b/sqlparser/README.md @@ -1,7 +1,7 @@ # sqlparser -An sql parser and static analyzer, written in pure Dart. At the moment, only `SELECT` statements -are supported. +An sql parser and static analyzer, written in pure Dart. At the moment, only `SELECT` and `DELETE` +statements are supported. Further, this library only targets the sqlite dialect at the time being. ## Features Not all features are available yet, put parsing select statements (even complex ones!) and @@ -56,6 +56,10 @@ resolvedColumns.map((c) => c.name)); // id, content, id, content, 3 + 4 resolvedColumns.map((c) => context.typeOf(c).type.type) // int, text, int, text, int, int ``` +## But why? +[Moor](https://pub.dev/packages/moor_flutter), a persistence library for Flutter apps, uses this +package to generate type-safe methods from sql. + ## Limitations - For now, only `SELECT` and `DELETE` expressions are implemented, `UPDATE` and `INSERT` will follow soon. diff --git a/sqlparser/pubspec.yaml b/sqlparser/pubspec.yaml index e1fc977a..f3eee5c3 100644 --- a/sqlparser/pubspec.yaml +++ b/sqlparser/pubspec.yaml @@ -10,6 +10,7 @@ environment: sdk: '>=2.2.2 <3.0.0' meta: ^1.1.7 collection: ^1.14.11 + source_span: ^1.5.5 dev_dependencies: test: ^1.0.0