diff --git a/docs/content/en/docs/Advanced Features/builder_options.md b/docs/content/en/docs/Advanced Features/builder_options.md index fb92bd57..d25314b3 100644 --- a/docs/content/en/docs/Advanced Features/builder_options.md +++ b/docs/content/en/docs/Advanced Features/builder_options.md @@ -68,6 +68,9 @@ At the moment, moor supports these options: columns back to null (by using `Value(null)`). Passing `null` was ignored before, making it impossible to set columns to `null`. * `named_parameters`: Generates named parameters for named variables in SQL queries. +* `new_sql_code_generation`: Generates SQL statements from the parsed AST instead of replacing substrings. This will also remove + unecessary whitespace and comments. + If enabling this option breaks your queries, please file an issue! ## Available extensions @@ -103,6 +106,7 @@ At the moment, they're opt-in to not break existing users. These options are: - `apply_converters_on_variables` - `generate_values_in_copy_with` +- `new_sql_code_generation` We recommend enabling these options. diff --git a/moor/build.yaml b/moor/build.yaml index 36318f93..41a1b2d4 100644 --- a/moor/build.yaml +++ b/moor/build.yaml @@ -12,6 +12,7 @@ targets: apply_converters_on_variables: true generate_values_in_copy_with: true named_parameters: true + new_sql_code_generation: true sqlite_modules: - json1 - fts5 \ No newline at end of file diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index b2b4b3bf..9380cb7c 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -967,7 +967,7 @@ abstract class _$Database extends GeneratedDatabase { $IngredientInRecipesTable(this); Selectable totalWeight() { return customSelect( - 'SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id', + 'SELECT r.title, SUM(ir.amount)AS total_weight FROM recipes AS r INNER JOIN recipe_ingredients AS ir ON ir.recipe = r.id GROUP BY r.id', variables: [], readsFrom: {recipes, ingredientInRecipes}).map((QueryRow row) { return TotalWeightResult( diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index d70347d0..4041499f 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -1576,7 +1576,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { late final WeirdTable weirdTable = WeirdTable(this); Selectable readConfig(String var1) { return customSelect( - 'SELECT\n config_key AS ck,\n config_value as cf,\n sync_state AS cs1,\n sync_state_implicit AS cs2\nFROM config WHERE config_key = ?', + 'SELECT config_key AS ck, config_value AS cf, sync_state AS cs1, sync_state_implicit AS cs2 FROM config WHERE config_key = ?', variables: [ Variable(var1) ], @@ -1598,7 +1598,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { final generatedclause = $write(clause); $arrayStartIndex += generatedclause.amountOfVariables; return customSelect( - 'SELECT * FROM config WHERE config_key IN ($expandedvar1) ${generatedclause.sql}', + 'SELECT * FROM config WHERE config_key IN($expandedvar1)${generatedclause.sql}', variables: [ for (var $ in var1) Variable($), ...generatedclause.introducedVariables @@ -1611,7 +1611,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable readDynamic( {Expression predicate = const CustomExpression('(TRUE)')}) { final generatedpredicate = $write(predicate); - return customSelect('SELECT * FROM config WHERE ${generatedpredicate.sql}', + return customSelect('SELECT * FROM config WHERE${generatedpredicate.sql}', variables: [...generatedpredicate.introducedVariables], readsFrom: {config}).map(config.mapFromRow); } @@ -1621,7 +1621,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { final expandedvar2 = $expandVar($arrayStartIndex, var2.length); $arrayStartIndex += var2.length; return customSelect( - 'SELECT config_key FROM config WHERE sync_state = ? OR sync_state_implicit IN ($expandedvar2)', + 'SELECT config_key FROM config WHERE sync_state = ? OR sync_state_implicit IN($expandedvar2)', variables: [ Variable(ConfigTable.$converter0.mapToSql(var1)), for (var $ in var2) @@ -1634,7 +1634,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable tableValued() { return customSelect( - 'SELECT "key", "value"\n FROM config, json_each(config.config_value)\n WHERE json_valid(config_value)', + 'SELECT "key", value FROM config,json_each(config.config_value)WHERE json_valid(config_value)', variables: [], readsFrom: {config}).map((QueryRow row) { return JsonResult( @@ -1647,7 +1647,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable another() { return customSelect( - 'SELECT \'one\' AS "key", NULLIF(\'two\', \'another\') AS "value"', + 'SELECT \'one\' AS "key", NULLIF(\'two\', \'another\')AS value', variables: [], readsFrom: {}).map((QueryRow row) { return JsonResult( @@ -1661,7 +1661,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable multiple({required Expression predicate}) { final generatedpredicate = $write(predicate, hasMultipleTables: true); return customSelect( - 'SELECT d.*, "c"."a" AS "nested_0.a", "c"."b" AS "nested_0.b", "c"."c" AS "nested_0.c" FROM with_constraints c\n INNER JOIN with_defaults d\n ON d.a = c.a AND d.b = c.b\n WHERE ${generatedpredicate.sql}', + 'SELECT d.*,"c"."a" AS "nested_0.a", "c"."b" AS "nested_0.b", "c"."c" AS "nested_0.c" FROM with_constraints AS c INNER JOIN with_defaults AS d ON d.a = c.a AND d.b = c.b WHERE${generatedpredicate.sql}', variables: [...generatedpredicate.introducedVariables], readsFrom: {withConstraints, withDefaults}).map((QueryRow row) { return MultipleResult( @@ -1683,7 +1683,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable readRowId({required Expression expr}) { final generatedexpr = $write(expr); return customSelect( - 'SELECT oid, * FROM config WHERE _rowid_ = ${generatedexpr.sql}', + 'SELECT oid, * FROM config WHERE _rowid_ =${generatedexpr.sql}', variables: [...generatedexpr.introducedVariables], readsFrom: {config}).map((QueryRow row) { return ReadRowIdResult( @@ -1700,7 +1700,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable cfeTest() { return customSelect( - 'WITH RECURSIVE\n cnt(x) AS (\n SELECT 1\n UNION ALL\n SELECT x+1 FROM cnt\n LIMIT 1000000\n )\n SELECT x FROM cnt', + 'WITH RECURSIVE cnt(x)AS (SELECT 1 UNION ALL SELECT x + 1 FROM cnt LIMIT 1000000) SELECT x FROM cnt', variables: [], readsFrom: {}).map((QueryRow row) => row.readInt('x')); } diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index f62fd11b..fe77a634 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1485,7 +1485,7 @@ abstract class _$TodoDb extends GeneratedDatabase { late final SomeDao someDao = SomeDao(this as TodoDb); Selectable allTodosWithCategory() { return customSelect( - 'SELECT t.*, c.id as catId, c."desc" as catDesc FROM todos t INNER JOIN categories c ON c.id = t.category', + 'SELECT t.*, c.id AS catId, c."desc" AS catDesc FROM todos AS t INNER JOIN categories AS c ON c.id = t.category', variables: [], readsFrom: {categories, todosTable}).map((QueryRow row) { return AllTodosWithCategoryResult( @@ -1515,7 +1515,7 @@ abstract class _$TodoDb extends GeneratedDatabase { final expandedvar3 = $expandVar($arrayStartIndex, var3.length); $arrayStartIndex += var3.length; return customSelect( - 'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1', + 'SELECT * FROM todos WHERE title = ?2 OR id IN($expandedvar3) OR title = ?1', variables: [ Variable(var1), Variable(var2), @@ -1620,7 +1620,7 @@ mixin _$SomeDaoMixin on DatabaseAccessor { $TodosTableTable get todosTable => attachedDatabase.todosTable; Selectable todosForUser({required int user}) { return customSelect( - 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', + 'SELECT t.* FROM todos AS t INNER JOIN shared_todos AS st ON st.todo = t.id INNER JOIN users AS u ON u.id = st.user WHERE u.id = :user', variables: [Variable(user)], readsFrom: {todosTable, sharedTodos, users}).map(todosTable.mapFromRow); } diff --git a/moor_generator/CHANGELOG.md b/moor_generator/CHANGELOG.md index 286d107f..15a03d7f 100644 --- a/moor_generator/CHANGELOG.md +++ b/moor_generator/CHANGELOG.md @@ -2,6 +2,10 @@ - Remove the `legacy_type_inference` option - Support moor 4 +- Add the `new_sql_code_generation` option to generate compiled SQL queries + (from moor files and annotations) based on the parsed AST. + Please consider enabling this option and reporting issues! + It will eventually become the default. ## 3.4.0 diff --git a/moor_generator/lib/src/analyzer/options.dart b/moor_generator/lib/src/analyzer/options.dart index 67605ced..12e2b617 100644 --- a/moor_generator/lib/src/analyzer/options.dart +++ b/moor_generator/lib/src/analyzer/options.dart @@ -83,6 +83,9 @@ class MoorOptions { @JsonKey(name: 'named_parameters', defaultValue: false) final bool generateNamedParameters; + @JsonKey(name: 'new_sql_code_generation', defaultValue: false) + final bool newSqlCodeGeneration; + const MoorOptions({ this.generateFromJsonStringConstructor = false, this.overrideHashAndEqualsInResultSets = false, @@ -98,6 +101,7 @@ class MoorOptions { this.applyConvertersOnVariables = false, this.generateValuesInCopyWith = false, this.generateNamedParameters = false, + this.newSqlCodeGeneration = false, this.modules = const [], }); diff --git a/moor_generator/lib/src/analyzer/options.g.dart b/moor_generator/lib/src/analyzer/options.g.dart index b4c2e832..19b45025 100644 --- a/moor_generator/lib/src/analyzer/options.g.dart +++ b/moor_generator/lib/src/analyzer/options.g.dart @@ -24,6 +24,7 @@ MoorOptions _$MoorOptionsFromJson(Map json) { 'apply_converters_on_variables', 'generate_values_in_copy_with', 'named_parameters', + 'new_sql_code_generation', ]); final val = MoorOptions( generateFromJsonStringConstructor: $checkedConvert( @@ -68,6 +69,9 @@ MoorOptions _$MoorOptionsFromJson(Map json) { false, generateNamedParameters: $checkedConvert(json, 'named_parameters', (v) => v as bool) ?? false, + newSqlCodeGeneration: + $checkedConvert(json, 'new_sql_code_generation', (v) => v as bool) ?? + false, modules: $checkedConvert( json, 'sqlite_modules', @@ -94,6 +98,7 @@ MoorOptions _$MoorOptionsFromJson(Map json) { 'rawResultSetData': 'raw_result_set_data', 'generateValuesInCopyWith': 'generate_values_in_copy_with', 'generateNamedParameters': 'named_parameters', + 'newSqlCodeGeneration': 'new_sql_code_generation', }); } diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 138d02c0..955ec612 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -9,6 +9,8 @@ import 'package:recase/recase.dart'; import 'package:sqlparser/sqlparser.dart' hide ResultColumn; import 'package:sqlparser/utils/node_to_text.dart'; +import 'sql_writer.dart'; + const highestAssignedIndexVar = '\$arrayStartIndex'; int _compareNodes(AstNode a, AstNode b) => @@ -33,22 +35,6 @@ class QueryWriter { _buffer = scope.leaf(); } - /// The expanded sql that we insert into queries whenever an array variable - /// appears. For the query "SELECT * FROM t WHERE x IN ?", we generate - /// ```dart - /// test(List var1) { - /// final expandedvar1 = List.filled(var1.length, '?').join(','); - /// customSelect('SELECT * FROM t WHERE x IN ($expandedvar1)', ...); - /// } - /// ``` - String _expandedName(FoundVariable v) { - return 'expanded${v.dartParameterName}'; - } - - String _placeholderContextName(FoundDartPlaceholder placeholder) { - return 'generated${placeholder.name}'; - } - void write() { if (query is SqlSelectQuery) { final select = query as SqlSelectQuery; @@ -370,7 +356,7 @@ class QueryWriter { // final expandedvar1 = $expandVar(, ); _buffer ..write('final ') - ..write(_expandedName(element)) + ..write(expandedName(element)) ..write(' = ') ..write(r'$expandVar(') ..write(highestAssignedIndexVar) @@ -390,7 +376,7 @@ class QueryWriter { _buffer ..write('final ') - ..write(_placeholderContextName(element)) + ..write(placeholderContextName(element)) ..write(' = ') ..write(r'$write(') ..write(element.dartParameterName); @@ -404,7 +390,7 @@ class QueryWriter { // similar to the case for expanded array variables, we need to // increase the index _increaseIndexCounter( - '${_placeholderContextName(element)}.amountOfVariables'); + '${placeholderContextName(element)}.amountOfVariables'); } } } @@ -456,8 +442,8 @@ class QueryWriter { _buffer.write('${constructVar(name)}'); } } else if (element is FoundDartPlaceholder) { - _buffer.write( - '...${_placeholderContextName(element)}.introducedVariables'); + _buffer + .write('...${placeholderContextName(element)}.introducedVariables'); } } @@ -468,6 +454,14 @@ class QueryWriter { /// been expanded. For instance, 'SELECT * FROM t WHERE x IN ?' will be turned /// into 'SELECT * FROM t WHERE x IN ($expandedVar1)'. String _queryCode() { + if (scope.options.newSqlCodeGeneration) { + return SqlWriter(query).write(); + } else { + return _legacyQueryCode(); + } + } + + String _legacyQueryCode() { // sort variables and placeholders by the order in which they appear final toReplace = query.fromContext.root.allDescendants .where((node) => @@ -508,14 +502,14 @@ class QueryWriter { (f) => f.variable.resolvedIndex == rewriteTarget.resolvedIndex); if (moorVar.isArray) { - replaceNode(rewriteTarget, '(\$${_expandedName(moorVar)})'); + replaceNode(rewriteTarget, '(\$${expandedName(moorVar)})'); } } else if (rewriteTarget is DartPlaceholder) { final moorPlaceholder = query.placeholders.singleWhere((p) => p.astNode == rewriteTarget); replaceNode(rewriteTarget, - '\${${_placeholderContextName(moorPlaceholder)}.sql}'); + '\${${placeholderContextName(moorPlaceholder)}.sql}'); } else if (rewriteTarget is NestedStarResultColumn) { final result = doubleStarColumnToResolvedTable[rewriteTarget]; if (result == null) continue; diff --git a/moor_generator/lib/src/writer/queries/sql_writer.dart b/moor_generator/lib/src/writer/queries/sql_writer.dart new file mode 100644 index 00000000..3ea65160 --- /dev/null +++ b/moor_generator/lib/src/writer/queries/sql_writer.dart @@ -0,0 +1,157 @@ +import 'package:charcode/ascii.dart'; +import 'package:moor_generator/moor_generator.dart'; +import 'package:moor_generator/src/utils/string_escaper.dart'; +import 'package:sqlparser/sqlparser.dart'; +import 'package:sqlparser/utils/node_to_text.dart'; + +/// The expanded sql that we insert into queries whenever an array variable +/// appears. For the query "SELECT * FROM t WHERE x IN ?", we generate +/// ```dart +/// test(List var1) { +/// final expandedvar1 = List.filled(var1.length, '?').join(','); +/// customSelect('SELECT * FROM t WHERE x IN ($expandedvar1)', ...); +/// } +/// ``` +String expandedName(FoundVariable v) { + return 'expanded${v.dartParameterName}'; +} + +String placeholderContextName(FoundDartPlaceholder placeholder) { + return 'generated${placeholder.name}'; +} + +class SqlWriter extends NodeSqlBuilder { + final StringBuffer _out; + final SqlQuery query; + final Map _starColumnToResolved; + + SqlWriter._(this.query, this._starColumnToResolved, StringBuffer out) + : _out = out, + super(_DartEscapingSink(out)); + + factory SqlWriter(SqlQuery query) { + // Index nested results by their syntactic origin for faster lookups later + var doubleStarColumnToResolvedTable = + const {}; + + if (query is SqlSelectQuery) { + doubleStarColumnToResolvedTable = { + for (final nestedResult in query.resultSet.nestedResults) + nestedResult.from: nestedResult + }; + } + return SqlWriter._(query, doubleStarColumnToResolvedTable, StringBuffer()); + } + + String write() { + _out.write("'"); + visit(query.fromContext.root, null); + _out.write("'"); + + return _out.toString(); + } + + FoundVariable _findMoorVar(Variable target) { + return query.variables.singleWhere( + (f) => f.variable.resolvedIndex == target.resolvedIndex, + orElse: () => null, + ); + } + + void _writeArrayVariable(FoundVariable moorVar) { + assert(moorVar.isArray); + _out.write('(\$${expandedName(moorVar)})'); + } + + @override + void visitDartPlaceholder(DartPlaceholder e, void arg) { + final moorPlaceholder = + query.placeholders.singleWhere((p) => p.astNode == e); + + _out.write('\${${placeholderContextName(moorPlaceholder)}.sql}'); + } + + @override + void visitNamedVariable(ColonNamedVariable e, void arg) { + final moor = _findMoorVar(e); + if (moor != null && moor.isArray) { + _writeArrayVariable(moor); + } else { + super.visitNamedVariable(e, arg); + } + } + + @override + void visitNumberedVariable(NumberedVariable e, void arg) { + final moor = _findMoorVar(e); + if (moor != null && moor.isArray) { + _writeArrayVariable(moor); + } else { + super.visitNumberedVariable(e, arg); + } + } + + @override + void visitMoorNestedStarResultColumn(NestedStarResultColumn e, void arg) { + final result = _starColumnToResolved[e]; + if (result == null) { + return super.visitMoorNestedStarResultColumn(e, arg); + } + + final select = query as SqlSelectQuery; + final prefix = select.resultSet.nestedPrefixFor(result); + final table = e.tableName; + + // Convert foo.** to "foo.a" AS "nested_0.a", ... for all columns in foo + var isFirst = true; + + for (final column in result.table.columns) { + if (isFirst) { + isFirst = false; + } else { + _out.write(', '); + } + + final columnName = column.name.name; + _out.write('"$table"."$columnName" AS "$prefix.$columnName"'); + } + } +} + +class _DartEscapingSink implements StringSink { + final StringSink _inner; + + _DartEscapingSink(this._inner); + + @override + void write(Object obj) { + _inner.write(escapeForDart(obj.toString())); + } + + @override + void writeAll(Iterable objects, [String separator = '']) { + var first = true; + for (final obj in objects) { + if (!first) write(separator); + + write(obj); + first = false; + } + } + + @override + void writeCharCode(int charCode) { + const needsEscape = {$$, $single_quote}; + if (needsEscape.contains(charCode)) { + _inner.writeCharCode($backslash); + } + + _inner.writeCharCode(charCode); + } + + @override + void writeln([Object obj = '']) { + write(obj); + writeCharCode($lf); + } +} diff --git a/moor_generator/test/writer/queries/sql_writer_test.dart b/moor_generator/test/writer/queries/sql_writer_test.dart new file mode 100644 index 00000000..5af7d776 --- /dev/null +++ b/moor_generator/test/writer/queries/sql_writer_test.dart @@ -0,0 +1,29 @@ +import 'package:moor_generator/moor_generator.dart'; +import 'package:moor_generator/src/writer/queries/sql_writer.dart'; +import 'package:sqlparser/sqlparser.dart'; +import 'package:test/test.dart'; + +void main() { + void check(String sql, String expectedDart) { + final engine = SqlEngine(); + final context = engine.analyze(sql); + final query = SqlSelectQuery( + 'name', context, [], [], InferredResultSet(null, []), null); + + final result = SqlWriter(query).write(); + + expect(result, expectedDart); + } + + test('removes unnecessary whitespace', () { + check(r'SELECT 1 + 3 AS r', r"'SELECT 1 + 3 AS r'"); + }); + + test('removes comments', () { + check(r'SELECT /*comment*/ 1', r"'SELECT 1'"); + }); + + test('escapes Dart characters in SQL', () { + check(r"SELECT '$hey';", r"'SELECT \'\$hey\''"); + }); +} diff --git a/sqlparser/lib/utils/node_to_text.dart b/sqlparser/lib/utils/node_to_text.dart index c084f1f3..cc67f846 100644 --- a/sqlparser/lib/utils/node_to_text.dart +++ b/sqlparser/lib/utils/node_to_text.dart @@ -20,16 +20,18 @@ extension NodeToText on AstNode { /// ways to represent an equivalent node (e.g. the no-op `FOR EACH ROW` on /// triggers). String toSql() { - final builder = _NodeSqlBuilder(); + final builder = NodeSqlBuilder(); builder.visit(this, null); - return builder._buffer.toString(); + return builder.buffer.toString(); } } -class _NodeSqlBuilder extends AstVisitor { - final StringBuffer _buffer = StringBuffer(); +class NodeSqlBuilder extends AstVisitor { + final StringSink buffer; bool _needsSpace = false; + NodeSqlBuilder([StringSink? buffer]) : buffer = buffer ?? StringBuffer(); + void _join(Iterable nodes, String separatingSymbol) { var isFirst = true; @@ -64,7 +66,7 @@ class _NodeSqlBuilder extends AstVisitor { _symbol(reverseKeywords[type]!, spaceAfter: true, spaceBefore: true); } - void _space() => _buffer.writeCharCode($space); + void _space() => buffer.writeCharCode($space); void _stringLiteral(String content) { _symbol("'$content'", spaceBefore: true, spaceAfter: true); @@ -76,7 +78,7 @@ class _NodeSqlBuilder extends AstVisitor { _space(); } - _buffer.write(lexeme); + buffer.write(lexeme); _needsSpace = spaceAfter; } @@ -636,10 +638,11 @@ class _NodeSqlBuilder extends AstVisitor { @override void visitInExpression(InExpression e, void arg) { visit(e.left, arg); - _keyword(TokenType.$is); + if (e.not) { _keyword(TokenType.not); } + _keyword(TokenType.$in); visit(e.inside, arg); } @@ -802,7 +805,7 @@ class _NodeSqlBuilder extends AstVisitor { void visitMoorFile(MoorFile e, void arg) { for (final stmt in e.statements) { visit(stmt, arg); - _buffer.write('\n'); + buffer.write('\n'); _needsSpace = false; } } diff --git a/sqlparser/test/utils/node_to_text_test.dart b/sqlparser/test/utils/node_to_text_test.dart index f9b6a2b2..b1a7f49e 100644 --- a/sqlparser/test/utils/node_to_text_test.dart +++ b/sqlparser/test/utils/node_to_text_test.dart @@ -252,6 +252,11 @@ CREATE UNIQUE INDEX my_idx ON t1 (c1, c2, c3) WHERE c1 < c3; testFormat('SELECT x AND y'); }); + test('in', () { + testFormat('SELECT x IN (SELECT * FROM foo);'); + testFormat('SELECT x NOT IN (SELECT * FROM foo);'); + }); + test('boolean literals', () { testFormat('SELECT TRUE OR FALSE'); });