Generate code for array variables in compiled statements

moor/CHANGELOG.md

@@ -3,6 +3,7 @@
 - Date time columns are now comparable
 - Make transactions easier to use: Thanks to some Dart async magic, methods called on your
   database object in a transaction callback will automatically be called on the transaction object.
+- Support list parameters in compiled custom queries (`SELECT * FROM entries WHERE id IN ?`)
 
 ## 1.5.1
 - Fixed an issue where transformed streams would not always update

moor/test/custom_queries_test.dart

@@ -0,0 +1,28 @@
+import 'package:test_api/test_api.dart';
+
+import 'data/tables/todos.dart';
+import 'data/utils/mocks.dart';
+
+void main() {
+  TodoDb db;
+  MockExecutor executor;
+
+  setUp(() {
+    executor = MockExecutor();
+    db = TodoDb(executor);
+  });
+
+  group('compiled custom queries', () {
+    // defined query: SELECT * FROM todos WHERE title = ?2 OR id IN ? OR title = ?1
+    test('work with arrays', () async {
+      await db.withIn('one', 'two', [1, 2, 3]);
+
+      verify(
+        executor.runSelect(
+          'SELECT * FROM todos WHERE title = ?2 OR id IN (?,?,?) OR title = ?1',
+          ['one', 'two', 1, 2, 3],
+        ),
+      );
+    });
+  });
+}

moor/test/data/tables/todos.dart

@@ -71,6 +71,7 @@ class PureDefaults extends Table {
     'allTodosWithCategory': 'SELECT t.*, c.id as catId, c."desc" as catDesc '
         'FROM todos t INNER JOIN categories c ON c.id = t.category',
     'deleteTodoById': 'DELETE FROM todos WHERE id = ?',
+    'withIn': 'SELECT * FROM todos WHERE title = ?2 OR id IN ? OR title = ?1'
   },
 )
 class TodoDb extends _$TodoDb {

moor/test/data/tables/todos.g.dart

@@ -1216,12 +1216,50 @@ abstract class _$TodoDb extends GeneratedDatabase {
   }
 
   Future<int> deleteTodoById(int var1, {QueryEngine operateOn}) {
-    return (operateOn ?? this)
+    return (operateOn ?? this).customUpdate(
-        .customUpdate('DELETE FROM todos WHERE id = ?', variables: [
+      'DELETE FROM todos WHERE id = ?',
-      Variable.withInt(var1),
+      variables: [
-    ], updates: {
+        Variable.withInt(var1),
-      todosTable
+      ],
-    });
+      updates: {todosTable},
+    );
+  }
+
+  TodoEntry _rowToTodoEntry(QueryRow row) {
+    return TodoEntry(
+      id: row.readInt('id'),
+      title: row.readString('title'),
+      content: row.readString('content'),
+      targetDate: row.readDateTime('target_date'),
+      category: row.readInt('category'),
+    );
+  }
+
+  Future<List<TodoEntry>> withIn(String var1, String var2, List<int> var3,
+      {QueryEngine operateOn}) {
+    final expandedvar3 = List.filled(var3.length, '?').join(',');
+    return (operateOn ?? this).customSelect(
+        'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1',
+        variables: [
+          Variable.withString(var1),
+          Variable.withString(var2),
+          for (var $ in var3) Variable.withInt($),
+        ]).then((rows) => rows.map(_rowToTodoEntry).toList());
+  }
+
+  Stream<List<TodoEntry>> watchWithIn(
+      String var1, String var2, List<int> var3) {
+    final expandedvar3 = List.filled(var3.length, '?').join(',');
+    return customSelectStream(
+        'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1',
+        variables: [
+          Variable.withString(var1),
+          Variable.withString(var2),
+          for (var $ in var3) Variable.withInt($),
+        ],
+        readsFrom: {
+          todosTable
+        }).map((rows) => rows.map(_rowToTodoEntry).toList());
   }
 
   @override

moor_generator/lib/src/model/sql_query.dart

@@ -1,16 +1,18 @@
 import 'package:moor_generator/src/model/specified_column.dart';
 import 'package:moor_generator/src/model/specified_table.dart';
 import 'package:recase/recase.dart';
+import 'package:sqlparser/sqlparser.dart';
 
 final _illegalChars = RegExp(r'[^0-9a-zA-Z_]');
 final _leadingDigits = RegExp(r'^\d*');
 
 abstract class SqlQuery {
   final String name;
-  final String sql;
+  final AnalysisContext fromContext;
+  String get sql => fromContext.sql;
   final List<FoundVariable> variables;
 
-  SqlQuery(this.name, this.sql, this.variables);
+  SqlQuery(this.name, this.fromContext, this.variables);
 }
 
 class SqlSelectQuery extends SqlQuery {
@@ -24,17 +26,17 @@ class SqlSelectQuery extends SqlQuery {
     return '${ReCase(name).pascalCase}Result';
   }
 
-  SqlSelectQuery(String name, String sql, List<FoundVariable> variables,
+  SqlSelectQuery(String name, AnalysisContext fromContext,
-      this.readsFrom, this.resultSet)
+      List<FoundVariable> variables, this.readsFrom, this.resultSet)
-      : super(name, sql, variables);
+      : super(name, fromContext, variables);
 }
 
 class UpdatingQuery extends SqlQuery {
   final List<SpecifiedTable> updates;
 
-  UpdatingQuery(
+  UpdatingQuery(String name, AnalysisContext fromContext,
-      String name, String sql, List<FoundVariable> variables, this.updates)
+      List<FoundVariable> variables, this.updates)
-      : super(name, sql, variables);
+      : super(name, fromContext, variables);
 }
 
 class InferredResultSet {
@@ -95,9 +97,21 @@ class FoundVariable {
   int index;
   String name;
   final ColumnType type;
+  final Variable variable;
 
-  FoundVariable(this.index, this.name, this.type);
+  /// Whether this variable is an array, which will be expanded into multiple
+  /// variables at runtime. We only accept queries where no explicitly numbered
+  /// vars appear after an array. This means that we can expand array variables
+  /// without having to look at other variables.
+  final bool isArray;
 
-  String get dartParameterName =>
+  FoundVariable(this.index, this.name, this.type, this.variable, this.isArray);
-      name?.replaceAll(_illegalChars, '') ?? 'var$index';
+
+  String get dartParameterName {
+    if (name != null) {
+      return name.replaceAll(_illegalChars, '');
+    } else {
+      return 'var${variable.resolvedIndex}';
+    }
+  }
 }

moor_generator/lib/src/parser/sql/query_handler.dart

@@ -35,7 +35,7 @@ class QueryHandler {
     context.root.accept(updatedFinder);
     _foundTables = updatedFinder.foundTables;
 
-    return UpdatingQuery(name, context.sql, _foundVariables,
+    return UpdatingQuery(name, context, _foundVariables,
         _foundTables.map(mapper.tableToMoor).toList());
   }
 
@@ -46,7 +46,7 @@ class QueryHandler {
     final moorTables = _foundTables.map(mapper.tableToMoor).toList();
 
     return SqlSelectQuery(
-        name, context.sql, _foundVariables, moorTables, _inferResultSet());
+        name, context, _foundVariables, moorTables, _inferResultSet());
   }
 
   InferredResultSet _inferResultSet() {

moor_generator/lib/src/parser/sql/sql_parser.dart

@@ -47,7 +47,11 @@ class SqlParser {
         ));
       }
 
-      foundQueries.add(QueryHandler(name, context, _mapper).handle());
+      try {
+        foundQueries.add(QueryHandler(name, context, _mapper).handle());
+      } catch (e) {
+        print('Error while generating APIs for ${context.sql}: $e');
+      }
     });
   }
 }

moor_generator/lib/src/parser/sql/type_mapping.dart

@@ -74,6 +74,11 @@ class TypeMapper {
       ..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex));
 
     final foundVariables = <FoundVariable>[];
+    // we don't allow variables with an explicit index after an array. For
+    // instance: SELECT * FROM t WHERE id IN ? OR id = ?2. The reason this is
+    // not allowed is that we expand the first arg into multiple vars at runtime
+    // which would break the index.
+    var maxIndex = 999;
     var currentIndex = 0;
 
     for (var used in usedVars) {
@@ -83,9 +88,30 @@ class TypeMapper {
 
       currentIndex++;
       final name = (used is ColonNamedVariable) ? used.name : null;
-      final type = resolvedToMoor(ctx.typeOf(used).type);
+      final explicitIndex =
+          (used is NumberedVariable) ? used.explicitIndex : null;
+      final internalType = ctx.typeOf(used);
+      final type = resolvedToMoor(internalType.type);
+      final isArray = internalType.type?.isArray ?? false;
 
-      foundVariables.add(FoundVariable(currentIndex, name, type));
+      if (explicitIndex != null && currentIndex >= maxIndex) {
+        throw ArgumentError(
+            'Cannot have a variable with an index lower than that of an array '
+            'appearing after an array!');
+      }
+
+      foundVariables
+          .add(FoundVariable(currentIndex, name, type, used, isArray));
+
+      // arrays cannot be indexed explicitly because they're expanded into
+      // multiple variables when executor
+      if (isArray && explicitIndex != null) {
+        throw ArgumentError(
+            'Cannot use an array variable with an explicit index');
+      }
+      if (isArray) {
+        maxIndex = used.resolvedIndex;
+      }
     }
 
     return foundVariables;

moor_generator/lib/src/utils/string_escaper.dart

@@ -1,4 +1,11 @@
 String asDartLiteral(String value) {
-  final escaped = value.replaceAll("'", "\\'").replaceAll('\n', '\\n');
+  final escaped = escapeForDart(value);
   return "'$escaped'";
 }
+
+String escapeForDart(String value) {
+  return value
+      .replaceAll("'", "\\'")
+      .replaceAll('\$', '\\\$')
+      .replaceAll('\n', '\\n');
+}

moor_generator/lib/src/writer/query_writer.dart

@@ -2,6 +2,7 @@ import 'package:moor_generator/src/model/specified_column.dart';
 import 'package:moor_generator/src/model/sql_query.dart';
 import 'package:moor_generator/src/utils/string_escaper.dart';
 import 'package:recase/recase.dart';
+import 'package:sqlparser/sqlparser.dart';
 
 /// Writes the handling code for a query. The code emitted will be a method that
 /// should be included in a generated database or dao class.
@@ -12,6 +13,18 @@ class QueryWriter {
 
   QueryWriter(this.query);
 
+  /// The expanded sql that we insert into queries whenever an array variable
+  /// appears. For the query "SELECT * FROM t WHERE x IN ?", we generate
+  /// ```dart
+  /// test(List<int> var1) {
+  ///   final expandedvar1 = List.filled(var1.length, '?').join(',');
+  ///   customSelect('SELECT * FROM t WHERE x IN ($expandedvar1)', ...);
+  /// }
+  /// ```
+  String _expandedName(FoundVariable v) {
+    return 'expanded${v.dartParameterName}';
+  }
+
   void writeInto(StringBuffer buffer) {
     if (query is SqlSelectQuery) {
       _writeSelect(buffer);
@@ -50,10 +63,11 @@ class QueryWriter {
   void _writeOneTimeReader(StringBuffer buffer) {
     buffer.write('Future<List<${_select.resultClassName}>> ${query.name}(');
     _writeParameters(buffer);
+    buffer.write(') {\n');
+    _writeExpandedDeclarations(buffer);
     buffer
-      ..write(') {\n')
       ..write('return (operateOn ?? this).') // use custom engine, if set
-      ..write('customSelect(${asDartLiteral(query.sql)},');
+      ..write('customSelect(${_queryCode()},');
     _writeVariables(buffer);
     buffer
       ..write(')')
@@ -70,9 +84,10 @@ class QueryWriter {
     // be used in transaction or similar context, only on the main database
     // engine.
     _writeParameters(buffer, dontOverrideEngine: true);
-    buffer
+    buffer.write(') {\n');
-      ..write(') {\n')
+
-      ..write('return customSelectStream(${asDartLiteral(query.sql)},');
+    _writeExpandedDeclarations(buffer);
+    buffer..write('return customSelectStream(${_queryCode()},');
 
     _writeVariables(buffer);
     buffer.write(',');
@@ -92,23 +107,29 @@ class QueryWriter {
      */
     buffer.write('Future<int> ${query.name}(');
     _writeParameters(buffer);
+    buffer.write(') {\n');
+
+    _writeExpandedDeclarations(buffer);
     buffer
-      ..write(') {\n')
       ..write('return (operateOn ?? this).')
-      ..write('customUpdate(${asDartLiteral(query.sql)},');
+      ..write('customUpdate(${_queryCode()},');
 
     _writeVariables(buffer);
     buffer.write(',');
     _writeUpdates(buffer);
 
-    buffer..write(');\n}\n');
+    buffer..write(',);\n}\n');
   }
 
   void _writeParameters(StringBuffer buffer,
       {bool dontOverrideEngine = false}) {
-    final paramList = query.variables
+    final paramList = query.variables.map((v) {
-        .map((v) => '${dartTypeNames[v.type]} ${v.dartParameterName}')
+      var dartType = dartTypeNames[v.type];
-        .join(', ');
+      if (v.isArray) {
+        dartType = 'List<$dartType>';
+      }
+      return '$dartType ${v.dartParameterName}';
+    }).join(', ');
 
     buffer.write(paramList);
 
@@ -120,18 +141,75 @@ class QueryWriter {
     }
   }
 
+  void _writeExpandedDeclarations(StringBuffer buffer) {
+    for (var variable in query.variables) {
+      if (variable.isArray) {
+        // final expandedvar1 = List.filled(var1.length, '?').join(',');
+        buffer
+          ..write('final ')
+          ..write(_expandedName(variable))
+          ..write(' = ')
+          ..write('List.filled(')
+          ..write(variable.dartParameterName)
+          ..write(".length, '?').join(',');");
+      }
+    }
+  }
+
   void _writeVariables(StringBuffer buffer) {
     buffer..write('variables: [');
 
     for (var variable in query.variables) {
-      buffer
+      // for a regular variable: Variable.withInt(x),
-        ..write(createVariable[variable.type])
+      // for a list of vars: for (var $ in vars) Variable.withInt($),
-        ..write('(${variable.dartParameterName}),');
+      final constructor = createVariable[variable.type];
+      final name = variable.dartParameterName;
+
+      if (variable.isArray) {
+        buffer.write('for (var \$ in $name) $constructor(\$)');
+      } else {
+        buffer.write('$constructor($name)');
+      }
+
+      buffer.write(',');
     }
 
     buffer..write(']');
   }
 
+  /// Returns a Dart string literal representing the query after variables have
+  /// been expanded. For instance, 'SELECT * FROM t WHERE x IN ?' will be turned
+  /// into 'SELECT * FROM t WHERE x IN ($expandedVar1)'.
+  String _queryCode() {
+    // sort variables by the order in which they appear
+    final vars = query.fromContext.root.allDescendants
+        .whereType<Variable>()
+        .toList()
+          ..sort((a, b) => a.firstPosition.compareTo(b.firstPosition));
+
+    final buffer = StringBuffer("'");
+    var lastIndex = 0;
+
+    for (var sqlVar in vars) {
+      final moorVar = query.variables.singleWhere((f) => f.variable == sqlVar);
+      if (!moorVar.isArray) continue;
+
+      // write everything that comes before this var into the buffer
+      final currentIndex = sqlVar.firstPosition;
+      final queryPart = query.sql.substring(lastIndex, currentIndex);
+      buffer.write(escapeForDart(queryPart));
+      lastIndex = sqlVar.lastPosition;
+
+      // write the ($expandedVar) par
+      buffer.write('(\$${_expandedName(moorVar)})');
+    }
+
+    // write the final part after the last variable, plus the ending '
+    buffer..write(escapeForDart(query.sql.substring(lastIndex)))..write("'");
+
+    return buffer.toString();
+  }
+
   void _writeReadsFrom(StringBuffer buffer) {
     final from = _select.readsFrom.map((t) => t.tableFieldName).join(', ');
     buffer..write('readsFrom: {')..write(from)..write('}');