Strip whitespace and comments from generated queries

This commit is contained in:
Simon Binder 2021-01-23 21:03:03 +01:00
parent 2da9175a27
commit 4ba12c4868
No known key found for this signature in database
GPG Key ID: 7891917E4147B8C0
13 changed files with 250 additions and 44 deletions

View File

@ -68,6 +68,9 @@ At the moment, moor supports these options:
columns back to null (by using `Value(null)`). Passing `null` was ignored before, making it impossible to set columns
to `null`.
* `named_parameters`: Generates named parameters for named variables in SQL queries.
* `new_sql_code_generation`: Generates SQL statements from the parsed AST instead of replacing substrings. This will also remove
unecessary whitespace and comments.
If enabling this option breaks your queries, please file an issue!
## Available extensions
@ -103,6 +106,7 @@ At the moment, they're opt-in to not break existing users. These options are:
- `apply_converters_on_variables`
- `generate_values_in_copy_with`
- `new_sql_code_generation`
We recommend enabling these options.

View File

@ -12,6 +12,7 @@ targets:
apply_converters_on_variables: true
generate_values_in_copy_with: true
named_parameters: true
new_sql_code_generation: true
sqlite_modules:
- json1
- fts5

View File

@ -967,7 +967,7 @@ abstract class _$Database extends GeneratedDatabase {
$IngredientInRecipesTable(this);
Selectable<TotalWeightResult> totalWeight() {
return customSelect(
'SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id',
'SELECT r.title, SUM(ir.amount)AS total_weight FROM recipes AS r INNER JOIN recipe_ingredients AS ir ON ir.recipe = r.id GROUP BY r.id',
variables: [],
readsFrom: {recipes, ingredientInRecipes}).map((QueryRow row) {
return TotalWeightResult(

View File

@ -1576,7 +1576,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
late final WeirdTable weirdTable = WeirdTable(this);
Selectable<Config> readConfig(String var1) {
return customSelect(
'SELECT\n config_key AS ck,\n config_value as cf,\n sync_state AS cs1,\n sync_state_implicit AS cs2\nFROM config WHERE config_key = ?',
'SELECT config_key AS ck, config_value AS cf, sync_state AS cs1, sync_state_implicit AS cs2 FROM config WHERE config_key = ?',
variables: [
Variable<String>(var1)
],
@ -1598,7 +1598,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
final generatedclause = $write(clause);
$arrayStartIndex += generatedclause.amountOfVariables;
return customSelect(
'SELECT * FROM config WHERE config_key IN ($expandedvar1) ${generatedclause.sql}',
'SELECT * FROM config WHERE config_key IN($expandedvar1)${generatedclause.sql}',
variables: [
for (var $ in var1) Variable<String>($),
...generatedclause.introducedVariables
@ -1611,7 +1611,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
Selectable<Config> readDynamic(
{Expression<bool> predicate = const CustomExpression('(TRUE)')}) {
final generatedpredicate = $write(predicate);
return customSelect('SELECT * FROM config WHERE ${generatedpredicate.sql}',
return customSelect('SELECT * FROM config WHERE${generatedpredicate.sql}',
variables: [...generatedpredicate.introducedVariables],
readsFrom: {config}).map(config.mapFromRow);
}
@ -1621,7 +1621,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
final expandedvar2 = $expandVar($arrayStartIndex, var2.length);
$arrayStartIndex += var2.length;
return customSelect(
'SELECT config_key FROM config WHERE sync_state = ? OR sync_state_implicit IN ($expandedvar2)',
'SELECT config_key FROM config WHERE sync_state = ? OR sync_state_implicit IN($expandedvar2)',
variables: [
Variable<int?>(ConfigTable.$converter0.mapToSql(var1)),
for (var $ in var2)
@ -1634,7 +1634,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
Selectable<JsonResult> tableValued() {
return customSelect(
'SELECT "key", "value"\n FROM config, json_each(config.config_value)\n WHERE json_valid(config_value)',
'SELECT "key", value FROM config,json_each(config.config_value)WHERE json_valid(config_value)',
variables: [],
readsFrom: {config}).map((QueryRow row) {
return JsonResult(
@ -1647,7 +1647,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
Selectable<JsonResult> another() {
return customSelect(
'SELECT \'one\' AS "key", NULLIF(\'two\', \'another\') AS "value"',
'SELECT \'one\' AS "key", NULLIF(\'two\', \'another\')AS value',
variables: [],
readsFrom: {}).map((QueryRow row) {
return JsonResult(
@ -1661,7 +1661,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
Selectable<MultipleResult> multiple({required Expression<bool> predicate}) {
final generatedpredicate = $write(predicate, hasMultipleTables: true);
return customSelect(
'SELECT d.*, "c"."a" AS "nested_0.a", "c"."b" AS "nested_0.b", "c"."c" AS "nested_0.c" FROM with_constraints c\n INNER JOIN with_defaults d\n ON d.a = c.a AND d.b = c.b\n WHERE ${generatedpredicate.sql}',
'SELECT d.*,"c"."a" AS "nested_0.a", "c"."b" AS "nested_0.b", "c"."c" AS "nested_0.c" FROM with_constraints AS c INNER JOIN with_defaults AS d ON d.a = c.a AND d.b = c.b WHERE${generatedpredicate.sql}',
variables: [...generatedpredicate.introducedVariables],
readsFrom: {withConstraints, withDefaults}).map((QueryRow row) {
return MultipleResult(
@ -1683,7 +1683,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
Selectable<ReadRowIdResult> readRowId({required Expression<int> expr}) {
final generatedexpr = $write(expr);
return customSelect(
'SELECT oid, * FROM config WHERE _rowid_ = ${generatedexpr.sql}',
'SELECT oid, * FROM config WHERE _rowid_ =${generatedexpr.sql}',
variables: [...generatedexpr.introducedVariables],
readsFrom: {config}).map((QueryRow row) {
return ReadRowIdResult(
@ -1700,7 +1700,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase {
Selectable<int> cfeTest() {
return customSelect(
'WITH RECURSIVE\n cnt(x) AS (\n SELECT 1\n UNION ALL\n SELECT x+1 FROM cnt\n LIMIT 1000000\n )\n SELECT x FROM cnt',
'WITH RECURSIVE cnt(x)AS (SELECT 1 UNION ALL SELECT x + 1 FROM cnt LIMIT 1000000) SELECT x FROM cnt',
variables: [],
readsFrom: {}).map((QueryRow row) => row.readInt('x'));
}

View File

@ -1485,7 +1485,7 @@ abstract class _$TodoDb extends GeneratedDatabase {
late final SomeDao someDao = SomeDao(this as TodoDb);
Selectable<AllTodosWithCategoryResult> allTodosWithCategory() {
return customSelect(
'SELECT t.*, c.id as catId, c."desc" as catDesc FROM todos t INNER JOIN categories c ON c.id = t.category',
'SELECT t.*, c.id AS catId, c."desc" AS catDesc FROM todos AS t INNER JOIN categories AS c ON c.id = t.category',
variables: [],
readsFrom: {categories, todosTable}).map((QueryRow row) {
return AllTodosWithCategoryResult(
@ -1515,7 +1515,7 @@ abstract class _$TodoDb extends GeneratedDatabase {
final expandedvar3 = $expandVar($arrayStartIndex, var3.length);
$arrayStartIndex += var3.length;
return customSelect(
'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1',
'SELECT * FROM todos WHERE title = ?2 OR id IN($expandedvar3) OR title = ?1',
variables: [
Variable<String?>(var1),
Variable<String?>(var2),
@ -1620,7 +1620,7 @@ mixin _$SomeDaoMixin on DatabaseAccessor<TodoDb> {
$TodosTableTable get todosTable => attachedDatabase.todosTable;
Selectable<TodoEntry> todosForUser({required int user}) {
return customSelect(
'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user',
'SELECT t.* FROM todos AS t INNER JOIN shared_todos AS st ON st.todo = t.id INNER JOIN users AS u ON u.id = st.user WHERE u.id = :user',
variables: [Variable<int>(user)],
readsFrom: {todosTable, sharedTodos, users}).map(todosTable.mapFromRow);
}

View File

@ -2,6 +2,10 @@
- Remove the `legacy_type_inference` option
- Support moor 4
- Add the `new_sql_code_generation` option to generate compiled SQL queries
(from moor files and annotations) based on the parsed AST.
Please consider enabling this option and reporting issues!
It will eventually become the default.
## 3.4.0

View File

@ -83,6 +83,9 @@ class MoorOptions {
@JsonKey(name: 'named_parameters', defaultValue: false)
final bool generateNamedParameters;
@JsonKey(name: 'new_sql_code_generation', defaultValue: false)
final bool newSqlCodeGeneration;
const MoorOptions({
this.generateFromJsonStringConstructor = false,
this.overrideHashAndEqualsInResultSets = false,
@ -98,6 +101,7 @@ class MoorOptions {
this.applyConvertersOnVariables = false,
this.generateValuesInCopyWith = false,
this.generateNamedParameters = false,
this.newSqlCodeGeneration = false,
this.modules = const [],
});

View File

@ -24,6 +24,7 @@ MoorOptions _$MoorOptionsFromJson(Map<String, dynamic> json) {
'apply_converters_on_variables',
'generate_values_in_copy_with',
'named_parameters',
'new_sql_code_generation',
]);
final val = MoorOptions(
generateFromJsonStringConstructor: $checkedConvert(
@ -68,6 +69,9 @@ MoorOptions _$MoorOptionsFromJson(Map<String, dynamic> json) {
false,
generateNamedParameters:
$checkedConvert(json, 'named_parameters', (v) => v as bool) ?? false,
newSqlCodeGeneration:
$checkedConvert(json, 'new_sql_code_generation', (v) => v as bool) ??
false,
modules: $checkedConvert(
json,
'sqlite_modules',
@ -94,6 +98,7 @@ MoorOptions _$MoorOptionsFromJson(Map<String, dynamic> json) {
'rawResultSetData': 'raw_result_set_data',
'generateValuesInCopyWith': 'generate_values_in_copy_with',
'generateNamedParameters': 'named_parameters',
'newSqlCodeGeneration': 'new_sql_code_generation',
});
}

View File

@ -9,6 +9,8 @@ import 'package:recase/recase.dart';
import 'package:sqlparser/sqlparser.dart' hide ResultColumn;
import 'package:sqlparser/utils/node_to_text.dart';
import 'sql_writer.dart';
const highestAssignedIndexVar = '\$arrayStartIndex';
int _compareNodes(AstNode a, AstNode b) =>
@ -33,22 +35,6 @@ class QueryWriter {
_buffer = scope.leaf();
}
/// The expanded sql that we insert into queries whenever an array variable
/// appears. For the query "SELECT * FROM t WHERE x IN ?", we generate
/// ```dart
/// test(List<int> var1) {
/// final expandedvar1 = List.filled(var1.length, '?').join(',');
/// customSelect('SELECT * FROM t WHERE x IN ($expandedvar1)', ...);
/// }
/// ```
String _expandedName(FoundVariable v) {
return 'expanded${v.dartParameterName}';
}
String _placeholderContextName(FoundDartPlaceholder placeholder) {
return 'generated${placeholder.name}';
}
void write() {
if (query is SqlSelectQuery) {
final select = query as SqlSelectQuery;
@ -370,7 +356,7 @@ class QueryWriter {
// final expandedvar1 = $expandVar(<startIndex>, <amount>);
_buffer
..write('final ')
..write(_expandedName(element))
..write(expandedName(element))
..write(' = ')
..write(r'$expandVar(')
..write(highestAssignedIndexVar)
@ -390,7 +376,7 @@ class QueryWriter {
_buffer
..write('final ')
..write(_placeholderContextName(element))
..write(placeholderContextName(element))
..write(' = ')
..write(r'$write(')
..write(element.dartParameterName);
@ -404,7 +390,7 @@ class QueryWriter {
// similar to the case for expanded array variables, we need to
// increase the index
_increaseIndexCounter(
'${_placeholderContextName(element)}.amountOfVariables');
'${placeholderContextName(element)}.amountOfVariables');
}
}
}
@ -456,8 +442,8 @@ class QueryWriter {
_buffer.write('${constructVar(name)}');
}
} else if (element is FoundDartPlaceholder) {
_buffer.write(
'...${_placeholderContextName(element)}.introducedVariables');
_buffer
.write('...${placeholderContextName(element)}.introducedVariables');
}
}
@ -468,6 +454,14 @@ class QueryWriter {
/// been expanded. For instance, 'SELECT * FROM t WHERE x IN ?' will be turned
/// into 'SELECT * FROM t WHERE x IN ($expandedVar1)'.
String _queryCode() {
if (scope.options.newSqlCodeGeneration) {
return SqlWriter(query).write();
} else {
return _legacyQueryCode();
}
}
String _legacyQueryCode() {
// sort variables and placeholders by the order in which they appear
final toReplace = query.fromContext.root.allDescendants
.where((node) =>
@ -508,14 +502,14 @@ class QueryWriter {
(f) => f.variable.resolvedIndex == rewriteTarget.resolvedIndex);
if (moorVar.isArray) {
replaceNode(rewriteTarget, '(\$${_expandedName(moorVar)})');
replaceNode(rewriteTarget, '(\$${expandedName(moorVar)})');
}
} else if (rewriteTarget is DartPlaceholder) {
final moorPlaceholder =
query.placeholders.singleWhere((p) => p.astNode == rewriteTarget);
replaceNode(rewriteTarget,
'\${${_placeholderContextName(moorPlaceholder)}.sql}');
'\${${placeholderContextName(moorPlaceholder)}.sql}');
} else if (rewriteTarget is NestedStarResultColumn) {
final result = doubleStarColumnToResolvedTable[rewriteTarget];
if (result == null) continue;

View File

@ -0,0 +1,157 @@
import 'package:charcode/ascii.dart';
import 'package:moor_generator/moor_generator.dart';
import 'package:moor_generator/src/utils/string_escaper.dart';
import 'package:sqlparser/sqlparser.dart';
import 'package:sqlparser/utils/node_to_text.dart';
/// The expanded sql that we insert into queries whenever an array variable
/// appears. For the query "SELECT * FROM t WHERE x IN ?", we generate
/// ```dart
/// test(List<int> var1) {
/// final expandedvar1 = List.filled(var1.length, '?').join(',');
/// customSelect('SELECT * FROM t WHERE x IN ($expandedvar1)', ...);
/// }
/// ```
String expandedName(FoundVariable v) {
return 'expanded${v.dartParameterName}';
}
String placeholderContextName(FoundDartPlaceholder placeholder) {
return 'generated${placeholder.name}';
}
class SqlWriter extends NodeSqlBuilder {
final StringBuffer _out;
final SqlQuery query;
final Map<NestedStarResultColumn, NestedResultTable> _starColumnToResolved;
SqlWriter._(this.query, this._starColumnToResolved, StringBuffer out)
: _out = out,
super(_DartEscapingSink(out));
factory SqlWriter(SqlQuery query) {
// Index nested results by their syntactic origin for faster lookups later
var doubleStarColumnToResolvedTable =
const <NestedStarResultColumn, NestedResultTable>{};
if (query is SqlSelectQuery) {
doubleStarColumnToResolvedTable = {
for (final nestedResult in query.resultSet.nestedResults)
nestedResult.from: nestedResult
};
}
return SqlWriter._(query, doubleStarColumnToResolvedTable, StringBuffer());
}
String write() {
_out.write("'");
visit(query.fromContext.root, null);
_out.write("'");
return _out.toString();
}
FoundVariable _findMoorVar(Variable target) {
return query.variables.singleWhere(
(f) => f.variable.resolvedIndex == target.resolvedIndex,
orElse: () => null,
);
}
void _writeArrayVariable(FoundVariable moorVar) {
assert(moorVar.isArray);
_out.write('(\$${expandedName(moorVar)})');
}
@override
void visitDartPlaceholder(DartPlaceholder e, void arg) {
final moorPlaceholder =
query.placeholders.singleWhere((p) => p.astNode == e);
_out.write('\${${placeholderContextName(moorPlaceholder)}.sql}');
}
@override
void visitNamedVariable(ColonNamedVariable e, void arg) {
final moor = _findMoorVar(e);
if (moor != null && moor.isArray) {
_writeArrayVariable(moor);
} else {
super.visitNamedVariable(e, arg);
}
}
@override
void visitNumberedVariable(NumberedVariable e, void arg) {
final moor = _findMoorVar(e);
if (moor != null && moor.isArray) {
_writeArrayVariable(moor);
} else {
super.visitNumberedVariable(e, arg);
}
}
@override
void visitMoorNestedStarResultColumn(NestedStarResultColumn e, void arg) {
final result = _starColumnToResolved[e];
if (result == null) {
return super.visitMoorNestedStarResultColumn(e, arg);
}
final select = query as SqlSelectQuery;
final prefix = select.resultSet.nestedPrefixFor(result);
final table = e.tableName;
// Convert foo.** to "foo.a" AS "nested_0.a", ... for all columns in foo
var isFirst = true;
for (final column in result.table.columns) {
if (isFirst) {
isFirst = false;
} else {
_out.write(', ');
}
final columnName = column.name.name;
_out.write('"$table"."$columnName" AS "$prefix.$columnName"');
}
}
}
class _DartEscapingSink implements StringSink {
final StringSink _inner;
_DartEscapingSink(this._inner);
@override
void write(Object obj) {
_inner.write(escapeForDart(obj.toString()));
}
@override
void writeAll(Iterable objects, [String separator = '']) {
var first = true;
for (final obj in objects) {
if (!first) write(separator);
write(obj);
first = false;
}
}
@override
void writeCharCode(int charCode) {
const needsEscape = {$$, $single_quote};
if (needsEscape.contains(charCode)) {
_inner.writeCharCode($backslash);
}
_inner.writeCharCode(charCode);
}
@override
void writeln([Object obj = '']) {
write(obj);
writeCharCode($lf);
}
}

View File

@ -0,0 +1,29 @@
import 'package:moor_generator/moor_generator.dart';
import 'package:moor_generator/src/writer/queries/sql_writer.dart';
import 'package:sqlparser/sqlparser.dart';
import 'package:test/test.dart';
void main() {
void check(String sql, String expectedDart) {
final engine = SqlEngine();
final context = engine.analyze(sql);
final query = SqlSelectQuery(
'name', context, [], [], InferredResultSet(null, []), null);
final result = SqlWriter(query).write();
expect(result, expectedDart);
}
test('removes unnecessary whitespace', () {
check(r'SELECT 1 + 3 AS r', r"'SELECT 1 + 3 AS r'");
});
test('removes comments', () {
check(r'SELECT /*comment*/ 1', r"'SELECT 1'");
});
test('escapes Dart characters in SQL', () {
check(r"SELECT '$hey';", r"'SELECT \'\$hey\''");
});
}

View File

@ -20,16 +20,18 @@ extension NodeToText on AstNode {
/// ways to represent an equivalent node (e.g. the no-op `FOR EACH ROW` on
/// triggers).
String toSql() {
final builder = _NodeSqlBuilder();
final builder = NodeSqlBuilder();
builder.visit(this, null);
return builder._buffer.toString();
return builder.buffer.toString();
}
}
class _NodeSqlBuilder extends AstVisitor<void, void> {
final StringBuffer _buffer = StringBuffer();
class NodeSqlBuilder extends AstVisitor<void, void> {
final StringSink buffer;
bool _needsSpace = false;
NodeSqlBuilder([StringSink? buffer]) : buffer = buffer ?? StringBuffer();
void _join(Iterable<AstNode> nodes, String separatingSymbol) {
var isFirst = true;
@ -64,7 +66,7 @@ class _NodeSqlBuilder extends AstVisitor<void, void> {
_symbol(reverseKeywords[type]!, spaceAfter: true, spaceBefore: true);
}
void _space() => _buffer.writeCharCode($space);
void _space() => buffer.writeCharCode($space);
void _stringLiteral(String content) {
_symbol("'$content'", spaceBefore: true, spaceAfter: true);
@ -76,7 +78,7 @@ class _NodeSqlBuilder extends AstVisitor<void, void> {
_space();
}
_buffer.write(lexeme);
buffer.write(lexeme);
_needsSpace = spaceAfter;
}
@ -636,10 +638,11 @@ class _NodeSqlBuilder extends AstVisitor<void, void> {
@override
void visitInExpression(InExpression e, void arg) {
visit(e.left, arg);
_keyword(TokenType.$is);
if (e.not) {
_keyword(TokenType.not);
}
_keyword(TokenType.$in);
visit(e.inside, arg);
}
@ -802,7 +805,7 @@ class _NodeSqlBuilder extends AstVisitor<void, void> {
void visitMoorFile(MoorFile e, void arg) {
for (final stmt in e.statements) {
visit(stmt, arg);
_buffer.write('\n');
buffer.write('\n');
_needsSpace = false;
}
}

View File

@ -252,6 +252,11 @@ CREATE UNIQUE INDEX my_idx ON t1 (c1, c2, c3) WHERE c1 < c3;
testFormat('SELECT x AND y');
});
test('in', () {
testFormat('SELECT x IN (SELECT * FROM foo);');
testFormat('SELECT x NOT IN (SELECT * FROM foo);');
});
test('boolean literals', () {
testFormat('SELECT TRUE OR FALSE');
});