Support table-valued functions

docs/lib/snippets/queries/json.dart

@@ -0,0 +1,80 @@
+// #docregion existing
+import 'dart:convert';
+
+import 'package:drift/drift.dart';
+import 'package:drift/extensions/json1.dart';
+import 'package:json_annotation/json_annotation.dart';
+
+// #enddocregion existing
+import 'package:drift/native.dart';
+
+part 'json.g.dart';
+
+// #docregion existing
+@JsonSerializable()
+class ContactData {
+  final String name;
+  final List<String> phoneNumbers;
+
+  ContactData(this.name, this.phoneNumbers);
+
+  factory ContactData.fromJson(Map<String, Object?> json) =>
+      _$ContactDataFromJson(json);
+
+  Map<String, Object?> toJson() => _$ContactDataToJson(this);
+}
+// #enddocregion existing
+
+// #docregion contacts
+class _ContactsConverter extends TypeConverter<ContactData, String> {
+  @override
+  ContactData fromSql(String fromDb) {
+    return ContactData.fromJson(json.decode(fromDb) as Map<String, Object?>);
+  }
+
+  @override
+  String toSql(ContactData value) {
+    return json.encode(value.toJson());
+  }
+}
+
+class Contacts extends Table {
+  IntColumn get id => integer().autoIncrement()();
+  TextColumn get data => text().map(_ContactsConverter())();
+
+  TextColumn get name => text().generatedAs(data.jsonExtract(r'$.name'))();
+}
+// #enddocregion contacts
+
+// #docregion calls
+class Calls extends Table {
+  IntColumn get id => integer().autoIncrement()();
+  BoolColumn get incoming => boolean()();
+  TextColumn get phoneNumber => text()();
+  DateTimeColumn get callTime => dateTime()();
+}
+// #enddocregion calls
+
+@DriftDatabase(tables: [Contacts, Calls])
+class MyDatabase extends _$MyDatabase {
+  MyDatabase() : super(NativeDatabase.memory());
+
+  @override
+  int get schemaVersion => 1;
+
+  // #docregion calls-with-contacts
+  Future<List<(Call, Contact)>> callsWithContact() async {
+    final phoneNumbersForContact =
+        contacts.data.jsonEach(this, r'$.phoneNumbers');
+    final phoneNumberQuery = selectOnly(phoneNumbersForContact)
+      ..addColumns([phoneNumbersForContact.value]);
+
+    final query = select(calls).join(
+        [innerJoin(contacts, calls.phoneNumber.isInQuery(phoneNumberQuery))]);
+
+    return query
+        .map((row) => (row.readTable(calls), row.readTable(contacts)))
+        .get();
+  }
+  // #enddocregion calls-with-contacts
+}

docs/pages/docs/Advanced Features/joins.md

@@ -217,3 +217,37 @@ joining this select statement onto a larger one grouping by category:
 {% include "blocks/snippet" snippets = snippets name = 'subquery' %}
 
 Any statement can be used as a subquery. But be aware that, unlike [subquery expressions]({{ 'expressions.md#scalar-subqueries' | pageUrl }}), full subqueries can't use tables from the outer select statement.
+
+## JSON support
+
+{% assign json_snippet = 'package:drift_docs/snippets/queries/json.dart.excerpt.json' | readString | json_decode %}
+
+sqlite3 has great support for [JSON operators](https://sqlite.org/json1.html) that are also available
+in drift (under the additional `'package:drift/extensions/json1.dart'` import).
+JSON support is helpful when storing a dynamic structure that is best represented with JSON, or when
+you have an existing structure (perhaps because you're migrating from a document-based storage)
+that you need to support.
+
+As an example, consider a contact book application that started with a JSON structure to store
+contacts:
+
+{% include "blocks/snippet" snippets = json_snippet name = 'existing' %}
+
+To easily store this contact representation in a drift database, one could use a JSON column:
+
+{% include "blocks/snippet" snippets = json_snippet name = 'contacts' %}
+
+Note the `name` column as well: It uses `generatedAs` with the `jsonExtract` function to
+extract the `name` field from the JSON value on the fly.
+The full syntax for JSON path arguments is explained on the [sqlite3 website](https://sqlite.org/json1.html#path_arguments).
+
+To make the example more complex, let's look at another table storing a log of phone calls:
+
+{% include "blocks/snippet" snippets = json_snippet name = 'calls' %}
+
+Let's say we wanted to find the contact for each call, if there is any with a matching phone number.
+For this to be expressible in SQL, each `contacts` row would somehow have to be expanded into a row
+for each stored phone number.
+Luckily, the `json_each` function in sqlite3 can do exactly that, and drift exposes it:
+
+{% include "blocks/snippet" snippets = json_snippet name = 'calls-with-contacts' %}

drift/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.12.0-dev
+
+- Add support for table-valued functions in the Dart query builder.
+- Support `json_each` and `json_tree`.
+
 ## 2.11.1
 
 - Allow using `.read()` for a column added to a join from the table, fixing a

drift/lib/extensions/json1.dart

@@ -53,4 +53,119 @@ extension JsonExtensions on Expression<String> {
       Variable.withString(path),
     ]).dartCast<T>();
   }
+
+  /// Calls the `json_each` table-valued function on `this` string, optionally
+  /// using [path] as the root path.
+  ///
+  /// This can be used to join every element in a JSON structure to a drift
+  /// query.
+  ///
+  /// See also: The [sqlite3 documentation](https://sqlite.org/json1.html#jeach)
+  /// and [JsonTableFunction].
+  JsonTableFunction jsonEach(DatabaseConnectionUser database, [String? path]) {
+    return JsonTableFunction._(database, functionName: 'json_each', arguments: [
+      this,
+      if (path != null) Variable(path),
+    ]);
+  }
+
+  /// Calls the `json_tree` table-valued function on `this` string, optionally
+  /// using [path] as the root path.
+  ///
+  /// This can be used to join every element in a JSON structure to a drift
+  /// query.
+  ///
+  /// See also: The [sqlite3 documentation](https://sqlite.org/json1.html#jeach)
+  /// and [JsonTableFunction].
+  JsonTableFunction jsonTree(DatabaseConnectionUser database, [String? path]) {
+    return JsonTableFunction._(database, functionName: 'json_tree', arguments: [
+      this,
+      if (path != null) Variable(path),
+    ]);
+  }
+}
+
+/// Calls [json table-valued functions](https://sqlite.org/json1.html#jeach) in
+/// drift.
+///
+/// With [JsonExtensions.jsonEach] and [JsonExtensions.jsonTree], a JSON value
+/// can be used a table-like structure available in queries and joins.
+///
+/// For an example and more details, see the [drift documentation](https://drift.simonbinder.eu/docs/advanced-features/joins/#json-support)
+final class JsonTableFunction extends TableValuedFunction<JsonTableFunction> {
+  JsonTableFunction._(
+    super.attachedDatabase, {
+    required super.functionName,
+    required super.arguments,
+    super.alias,
+  }) : super(
+          columns: [
+            GeneratedColumn<DriftAny>('key', alias ?? functionName, true,
+                type: DriftSqlType.any),
+            GeneratedColumn<DriftAny>('value', alias ?? functionName, true,
+                type: DriftSqlType.any),
+            GeneratedColumn<String>('type', alias ?? functionName, true,
+                type: DriftSqlType.string),
+            GeneratedColumn<DriftAny>('atom', alias ?? functionName, true,
+                type: DriftSqlType.any),
+            GeneratedColumn<int>('id', alias ?? functionName, true,
+                type: DriftSqlType.int),
+            GeneratedColumn<int>('parent', alias ?? functionName, true,
+                type: DriftSqlType.int),
+            GeneratedColumn<String>('fullkey', alias ?? functionName, true,
+                type: DriftSqlType.string),
+            GeneratedColumn<String>('path', alias ?? functionName, true,
+                type: DriftSqlType.string),
+          ],
+        );
+
+  Expression<T> _col<T extends Object>(String name) {
+    return columnsByName[name]! as Expression<T>;
+  }
+
+  /// The JSON key under which this element can be found in its parent, or
+  /// `null` if this is the root element.
+  ///
+  /// Child elements of objects have a string key, elements in arrays are
+  /// represented by their index.
+  Expression<DriftAny> get key => _col('key');
+
+  /// The value for the current value.
+  ///
+  /// Scalar values are returned directly, objects and arrays are returned as
+  /// JSON strings.
+  Expression<DriftAny> get value => _col('value');
+
+  /// The result of calling [`sqlite3_type`](https://sqlite.org/json1.html#the_json_type_function)
+  /// on this JSON element.
+  Expression<String> get type => _col('type');
+
+  /// The [value], or `null` if this is not a scalar value (so either an object
+  /// or an array).
+  Expression<DriftAny> get atom => _col('atom');
+
+  /// An id uniquely identifying this element in the original JSON tree.
+  Expression<int> get id => _col('id');
+
+  /// The [id] of the parent of this element.
+  Expression<int> get parent => _col('parent');
+
+  /// The JSON key that can be passed to functions like
+  /// [JsonExtensions.jsonExtract] to find this value.
+  Expression<String> get fullKey => _col('fullkey');
+
+  /// Similar to [fullKey], but relative to the `root` argument passed to
+  /// [JsonExtensions.jsonEach] or [JsonExtensions.jsonTree].
+  Expression<String> get path => _col('path');
+
+  @override
+  ResultSetImplementation<JsonTableFunction, TypedResult> createAlias(
+      String alias) {
+    return JsonTableFunction._(
+      attachedDatabase,
+      functionName: entityName,
+      arguments: arguments,
+      alias: alias,
+    );
+  }
 }

drift/lib/src/runtime/query_builder/components/subquery.dart

@@ -121,15 +121,6 @@ class Subquery<Row> extends ResultSetImplementation<Subquery, Row>
 
   @override
   FutureOr<Row> map(Map<String, dynamic> data, {String? tablePrefix}) {
-    if (tablePrefix == null) {
-      return select._mapRow(data);
-    } else {
-      final withoutPrefix = {
-        for (final MapEntry(:key, :value) in columnsByName.entries)
-          key: data['$tablePrefix.$value']
-      };
-
-      return select._mapRow(withoutPrefix);
-    }
+    return select._mapRow(data.withoutPrefix(tablePrefix));
   }
 }

drift/lib/src/runtime/query_builder/components/table_valued_function.dart

@@ -0,0 +1,100 @@
+import 'dart:async';
+
+import 'package:meta/meta.dart';
+
+import '../../../dsl/dsl.dart';
+import '../../api/runtime_api.dart';
+import '../../utils.dart';
+import '../query_builder.dart';
+
+/// In sqlite3, a table-valued function is a function that resolves to a result
+/// set, meaning that it can be selected from.
+///
+/// For more information on table-valued functions in general, visit their
+/// [documentation](https://sqlite.org/vtab.html#tabfunc2) on the sqlite website.
+///
+/// This class is meant to be extended for each table-valued function, so that
+/// the [Self] type parameter points to the actual implementation class. The
+/// class must also implement [createAlias] correctly (ensuring that every
+/// column has its [GeneratedColumn.tableName] set to the [aliasedName]).
+///
+/// For an example of a table-valued function in drift, see the
+/// `JsonTableFunction` in `package:drift/json1.dart`. It makes the `json_each`
+/// and `json_tree` table-valued functions available to drift.
+@experimental
+abstract base class TableValuedFunction<Self extends ResultSetImplementation>
+    extends ResultSetImplementation<Self, TypedResult>
+    implements HasResultSet, Component {
+  final String _functionName;
+
+  /// The arguments passed to the table-valued function.
+  final List<Expression> arguments;
+
+  @override
+  final DatabaseConnectionUser attachedDatabase;
+
+  @override
+  final List<GeneratedColumn<Object>> $columns;
+
+  @override
+  final String aliasedName;
+
+  /// Constructor for table-valued functions.
+  ///
+  /// This takes the [attachedDatabase] (used to interpret results), the name
+  /// of the function as well as arguments passed to it and finally the schema
+  /// of the table (in the form of [columns]).
+  TableValuedFunction(
+    this.attachedDatabase, {
+    required String functionName,
+    required this.arguments,
+    required List<GeneratedColumn> columns,
+    String? alias,
+  })  : _functionName = functionName,
+        $columns = columns,
+        aliasedName = alias ?? functionName;
+
+  @override
+  Self get asDslTable => this as Self;
+
+  @override
+  late final Map<String, GeneratedColumn<Object>> columnsByName = {
+    for (final column in $columns) column.name: column,
+  };
+
+  @override
+  String get entityName => _functionName;
+
+  @override
+  FutureOr<TypedResult> map(Map<String, dynamic> data, {String? tablePrefix}) {
+    final row = QueryRow(data.withoutPrefix(tablePrefix), attachedDatabase);
+    return TypedResult(
+      const {},
+      row,
+      {
+        for (final column in $columns)
+          column: attachedDatabase.typeMapping
+              .read(column.type, row.data[column.name]),
+      },
+    );
+  }
+
+  @override
+  void writeInto(GenerationContext context) {
+    context.buffer
+      ..write(_functionName)
+      ..write('(');
+
+    var first = true;
+    for (final argument in arguments) {
+      if (!first) {
+        context.buffer.write(', ');
+      }
+
+      argument.writeInto(context);
+      first = false;
+    }
+
+    context.buffer.write(')');
+  }
+}

drift/lib/src/runtime/query_builder/helpers.dart

@@ -1,24 +1,31 @@
 @internal
 library;
 
-import 'package:drift/drift.dart';
-
 import 'package:meta/meta.dart';
 
+import 'query_builder.dart';
+
 /// Internal utilities for building queries that aren't exported.
 extension WriteDefinition on GenerationContext {
   /// Writes the result set to this context, suitable to implement `FROM`
   /// clauses and joins.
   void writeResultSet(ResultSetImplementation resultSet) {
-    if (resultSet is Subquery) {
-      buffer.write('(');
-      resultSet.select.writeInto(this);
-      buffer
-        ..write(') ')
-        ..write(resultSet.aliasedName);
-    } else {
-      buffer.write(resultSet.tableWithAlias);
-      watchedTables.add(resultSet);
+    switch (resultSet) {
+      case Subquery(:final select):
+        buffer.write('(');
+        select.writeInto(this);
+        buffer
+          ..write(') ')
+          ..write(resultSet.aliasedName);
+      case TableValuedFunction():
+        resultSet.writeInto(this);
+
+        if (resultSet.aliasedName != resultSet.entityName) {
+          buffer.write(' ${resultSet.aliasedName}');
+        }
+      default:
+        buffer.write(resultSet.tableWithAlias);
+        watchedTables.add(resultSet);
     }
   }
 

drift/lib/src/runtime/query_builder/query_builder.dart

@@ -21,6 +21,7 @@ import 'package:drift/src/utils/single_transformer.dart';
 import 'package:meta/meta.dart';
 
 import '../../utils/async.dart';
+import '../utils.dart';
 // New files should not be part of this mega library, which we're trying to
 // split up.
 
@@ -28,6 +29,7 @@ import 'expressions/case_when.dart';
 import 'expressions/internal.dart';
 import 'helpers.dart';
 
+export 'components/table_valued_function.dart';
 export 'expressions/bitwise.dart';
 export 'expressions/case_when.dart';
 export 'on_table.dart';

drift/lib/src/runtime/utils.dart

@@ -0,0 +1,18 @@
+/// Provides [withoutPrefix], a method useful to interpret SQL results.
+extension RemovePrefix on Map<String, Object?> {
+  /// Returns a map of all keys starting with the table prefix, but with the
+  /// prefix removed.
+  Map<String, Object?> withoutPrefix(String? tablePrefix) {
+    if (tablePrefix != null) {
+      final actualPrefix = '$tablePrefix.';
+      final prefixLength = actualPrefix.length;
+
+      return {
+        for (final MapEntry(:key, :value) in entries)
+          if (key.startsWith(actualPrefix)) key.substring(prefixLength): value,
+      };
+    } else {
+      return this;
+    }
+  }
+}

drift/test/database/statements/select_test.dart

@@ -1,6 +1,7 @@
 import 'dart:async';
 
 import 'package:drift/drift.dart' hide isNull;
+import 'package:drift/extensions/json1.dart';
 import 'package:mockito/mockito.dart';
 import 'package:test/test.dart';
 
@@ -240,4 +241,19 @@ void main() {
 
     verify(executor.runSelect('SELECT * FROM (SELECT * FROM "todos") s;', []));
   });
+
+  test('select from table-valued function', () async {
+    final each = db.todosTable.content.jsonEach(db, r'$.foo');
+
+    final query = db
+        .select(db.todosTable)
+        .join([innerJoin(each, each.atom.isNotNull(), useColumns: false)]);
+
+    await query.get();
+
+    verify(executor.runSelect(
+      'SELECT "todos"."id" AS "todos.id", "todos"."title" AS "todos.title", "todos"."content" AS "todos.content", "todos"."target_date" AS "todos.target_date", "todos"."category" AS "todos.category", "todos"."status" AS "todos.status" FROM "todos" INNER JOIN json_each("todos"."content", ?) ON "json_each"."atom" IS NOT NULL;',
+      [r'$.foo'],
+    ));
+  });
 }

drift/test/extensions/json1_integration_test.dart

@@ -1,7 +1,7 @@
 @Tags(['integration'])
 import 'dart:convert';
 
-import 'package:drift/drift.dart';
+import 'package:drift/drift.dart' hide isNull;
 import 'package:drift/extensions/json1.dart';
 import 'package:test/test.dart';
 
@@ -9,16 +9,21 @@ import '../generated/todos.dart';
 import '../test_utils/test_utils.dart';
 
 void main() {
+  const jsonObject = {
+    'foo': 'bar',
+    'array': [
+      'one',
+      'two',
+      'three',
+    ],
+  };
+
+  late TodoDb db;
+
+  setUp(() => db = TodoDb(testInMemoryDatabase()));
+  tearDown(() => db.close());
+
   test('json1 integration test', () async {
-    final db = TodoDb(testInMemoryDatabase());
-    const jsonObject = {
-      'foo': 'bar',
-      'array': [
-        'one',
-        'two',
-        'three',
-      ],
-    };
     await db.into(db.pureDefaults).insert(PureDefaultsCompanion(
         txt: Value(MyCustomObject(json.encode(jsonObject)))));
 
@@ -30,5 +35,44 @@ void main() {
 
     final resultRow = await query.getSingle();
     expect(resultRow.read(arrayLengthExpr), 3);
-  }, tags: const ['integration']);
+  });
+
+  test('json_each', () async {
+    final function = Variable<String>(json.encode(jsonObject)).jsonEach(db);
+    final rows = await db.select(function).get();
+
+    expect(rows, hasLength(2));
+
+    expect(rows[0].read(function.key), DriftAny('foo'));
+    expect(rows[0].read(function.value), DriftAny('bar'));
+    expect(rows[0].read(function.type), 'text');
+    expect(rows[0].read(function.atom), DriftAny('bar'));
+    expect(rows[0].read(function.id), 2);
+    expect(rows[0].read(function.parent), isNull);
+    expect(rows[0].read(function.fullKey), r'$.foo');
+    expect(rows[0].read(function.path), r'$');
+  });
+
+  test('json_tree', () async {
+    // Make sure we can use aliases as well
+    final function = Variable<String>(json.encode(jsonObject)).jsonTree(db);
+    final parent = db.alias(function, 'parent');
+
+    final query = db
+        .selectOnly(function)
+        .join([leftOuterJoin(parent, parent.id.equalsExp(function.parent))])
+      ..addColumns([function.atom, parent.id])
+      ..where(function.atom.isNotNull());
+
+    final rows = await query
+        .map((row) => (row.read(function.atom), row.read(parent.id)))
+        .get();
+
+    expect(rows, [
+      (DriftAny('bar'), 0),
+      (DriftAny('one'), 4),
+      (DriftAny('two'), 4),
+      (DriftAny('three'), 4),
+    ]);
+  });
 }