Add support for subqueries.

Closes #612.
This commit is contained in:
Simon Binder 2023-07-20 22:30:20 +02:00
parent e98da9cecb
commit 03d3c837be
No known key found for this signature in database
GPG Key ID: 7891917E4147B8C0
16 changed files with 433 additions and 50 deletions

View File

@ -107,4 +107,40 @@ extension GroupByQueries on MyDatabase {
});
}
// #enddocregion createCategoryForUnassignedTodoEntries
// #docregion subquery
Future<List<(Category, int)>> amountOfLengthyTodoItemsPerCategory() async {
final longestTodos = Subquery(
select(todos)
..orderBy([(row) => OrderingTerm.desc(row.title.length)])
..limit(10),
's',
);
// In the main query, we want to count how many entries in longestTodos were
// found for each category. But we can't access todos.title directly since
// we're not selecting from `todos`. Instead, we'll use Subquery.ref to read
// from a column in a subquery.
final itemCount = longestTodos.ref(todos.title).count();
final query = select(categories).join(
[
innerJoin(
longestTodos,
// Again using .ref() here to access the category in the outer select
// statement.
longestTodos.ref(todos.category).equalsExp(categories.id),
useColumns: false,
)
],
)
..addColumns([itemCount])
..groupBy([categories.id]);
final rows = await query.get();
return [
for (final row in rows) (row.readTable(categories), row.read(itemCount)!),
];
}
// #enddocregion subquery
}

View File

@ -243,6 +243,11 @@ any rows. For instance, we could use this to find empty categories:
{% include "blocks/snippet" snippets = snippets name = 'emptyCategories' %}
### Full subqueries
Drift also supports subqueries that appear in `JOIN`s, which are described in the
[documentation for joins]({{ 'joins.md#subqueries' | pageUrl }}).
## Custom expressions
If you want to inline custom sql into Dart queries, you can use a `CustomExpression` class.
It takes a `sql` parameter that lets you write custom expressions:

View File

@ -203,3 +203,17 @@ select statement.
In the example, the `newDescription` expression as added as a column to the query.
Then, the map entry `categories.description: newDescription` is used so that the `description` column
for new category rows gets set to that expression.
## Subqueries
Starting from drift 2.11, you can use `Subquery` to use an existing select statement as part of more
complex join.
This snippet uses `Subquery` to count how many of the top-10 todo items (by length of their title) are
in each category.
It does this by first creating a select statement for the top-10 items (but not executing it), and then
joining this select statement onto a larger one grouping by category:
{% include "blocks/snippet" snippets = snippets name = 'subquery' %}
Any statement can be used as a subquery. But be aware that, unlike [subquery expressions]({{ 'expressions.md#scalar-subqueries' | pageUrl }}), full subqueries can't use tables from the outer select statement.

View File

@ -1,3 +1,7 @@
## 2.11.0
- Add support for subqueries in the Dart query builder.
## 2.10.0
- Adds the `schema steps` command to `drift_dev`. It generates an API making it

View File

@ -58,8 +58,7 @@ class Join<T extends HasResultSet, D> extends Component {
context.buffer.write(' JOIN ');
final resultSet = table as ResultSetImplementation<T, D>;
context.buffer.write(resultSet.tableWithAlias);
context.watchedTables.add(resultSet);
context.writeResultSet(resultSet);
if (_type != _JoinType.cross) {
context.buffer.write(' ON ');

View File

@ -0,0 +1,135 @@
part of '../query_builder.dart';
/// A subquery allows reading from another complex query in a join.
///
/// An existing query can be constructed via [DatabaseConnectionUser.select] or
/// [DatabaseConnectionUser.selectOnly] and then wrapped in [Subquery] to be
/// used in another query.
///
/// For instance, assuming database storing todo items with optional categories
/// (through a reference from todo items to categories), this query uses a
/// subquery to count how many of the top-10 todo items (by length) are in each
/// category:
///
/// ```dart
/// final longestTodos = Subquery(
/// select(todosTable)
/// ..orderBy([(row) => OrderingTerm.desc(row.title.length)])
/// ..limit(10),
/// 's',
/// );
///
/// final itemCount = subquery.ref(todosTable.id).count();
/// final query = select(categories).join([
/// innerJoin(
/// longestTodos,
/// subquery.ref(todosTable.category).equalsExp(categories.id),
/// useColumns: false,
/// )])
/// ..groupBy([categories.id])
/// ..addColumns([itemCount]);
/// ```
///
/// Note that the column from the subquery (here, the id of a todo entry) is not
/// directly available in the outer query, it needs to be accessed through
/// [Subquery.ref].
/// Columns added to the top-level query (via [ref]) can be accessed directly
/// through [TypedResult.read]. When columns from a subquery are added to the
/// top-level select as well, [TypedResult.readTable] can be used to read an
/// entire row from the subquery. It returns a nested [TypedResult] for the
/// subquery.
///
/// See also: [subqueryExpression], for subqueries which only return one row and
/// one column.
class Subquery<Row> extends ResultSetImplementation<Subquery, Row>
implements HasResultSet {
/// The inner [select] statement of this subquery.
final BaseSelectStatement<Row> select;
@override
final String entityName;
/// Creates a subqery from the inner [select] statement forming the base of
/// the subquery and a unique name of this subquery in the statement being
/// executed.
Subquery(this.select, this.entityName);
/// Makes a column from the subquery available to the outer select statement.
///
/// For instance, consider a complex column like `subqueryContentLength` being
/// added into a subquery:
///
/// ```dart
/// final subqueryContentLength = todoEntries.content.length.sum();
/// final subquery = Subquery(
/// db.selectOnly(todoEntries)
/// ..addColumns([todoEntries.category, subqueryContentLength])
/// ..groupBy([todoEntries.category]),
/// 's');
/// ```
///
/// When the `subqueryContentLength` column gets written, drift will write
/// the actual `SUM()` expression which is only valid in the subquery itself.
/// When an outer query joining the subqery wants to read the column, it needs
/// to refer to that expression by name. This is what [ref] is doing:
///
/// ```dart
/// final readableLength = subquery.ref(subqueryContentLength);
/// final query = selectOnly(categories)
/// ..addColumns([categories.id, readableLength])
/// ..join([
/// innerJoin(subquery,
/// subquery.ref(db.todosTable.category).equalsExp(db.categories.id))
/// ]);
/// ```
///
/// Here, [ref] is called two times: Once to obtain a column selected by the
/// outer query and once as a join condition.
///
/// [ref] needs to be used every time a column from a subquery is used in an
/// outer query, regardless of the context.
Expression<T> ref<T extends Object>(Expression<T> inner) {
final name = select._nameForColumn(inner);
if (name == null) {
throw ArgumentError(
'The source select statement does not contain that column');
}
return columnsByName[name]!.dartCast();
}
@override
late final List<GeneratedColumn<Object>> $columns = [
for (final (expr, name) in select._expandedColumns)
GeneratedColumn(
name,
entityName,
true,
type: expr.driftSqlType,
),
];
@override
late final Map<String, GeneratedColumn<Object>> columnsByName = {
for (final column in $columns) column.name: column,
};
@override
Subquery get asDslTable => this;
@override
DatabaseConnectionUser get attachedDatabase => (select as Query).database;
@override
FutureOr<Row> map(Map<String, dynamic> data, {String? tablePrefix}) {
if (tablePrefix == null) {
return select._mapRow(data);
} else {
final withoutPrefix = {
for (final MapEntry(:key, :value) in columnsByName.entries)
key: data['$tablePrefix.$value']
};
return select._mapRow(withoutPrefix);
}
}
}

View File

@ -503,7 +503,7 @@ class FunctionCallExpression<R extends Object> extends Expression<R> {
}
void _checkSubquery(BaseSelectStatement statement) {
final columns = statement._returnedColumnCount;
final columns = statement._expandedColumns.length;
if (columns != 1) {
throw ArgumentError.value(statement, 'statement',
'Must return exactly one column (actually returns $columns)');

View File

@ -0,0 +1,19 @@
import 'package:drift/drift.dart';
/// Utilities for writing the definition of a result set into a query.
extension WriteDefinition on GenerationContext {
/// Writes the result set to this context, suitable to implement `FROM`
/// clauses and joins.
void writeResultSet(ResultSetImplementation resultSet) {
if (resultSet is Subquery) {
buffer.write('(');
resultSet.select.writeInto(this);
buffer
..write(') ')
..write(resultSet.aliasedName);
} else {
buffer.write(resultSet.tableWithAlias);
watchedTables.add(resultSet);
}
}
}

View File

@ -23,8 +23,10 @@ import 'package:meta/meta.dart';
import '../../utils/async.dart';
// New files should not be part of this mega library, which we're trying to
// split up.
import 'expressions/case_when.dart';
import 'expressions/internal.dart';
import 'helpers.dart';
export 'expressions/bitwise.dart';
export 'expressions/case_when.dart';
@ -34,6 +36,7 @@ part 'components/group_by.dart';
part 'components/join.dart';
part 'components/limit.dart';
part 'components/order_by.dart';
part 'components/subquery.dart';
part 'components/where.dart';
part 'expressions/aggregate.dart';
part 'expressions/algebra.dart';

View File

@ -8,12 +8,14 @@ typedef OrderClauseGenerator<T> = OrderingTerm Function(T tbl);
///
/// Users are not allowed to extend, implement or mix-in this class.
@sealed
abstract class BaseSelectStatement extends Component {
int get _returnedColumnCount;
abstract class BaseSelectStatement<Row> extends Component {
Iterable<(Expression, String)> get _expandedColumns;
/// The name for the given [expression] in the result set, or `null` if
/// [expression] was not added as a column to this select statement.
String? _nameForColumn(Expression expression);
FutureOr<Row> _mapRow(Map<String, Object?> fromDatabase);
}
/// A select statement that doesn't use joins.
@ -21,7 +23,7 @@ abstract class BaseSelectStatement extends Component {
/// For more information, see [DatabaseConnectionUser.select].
class SimpleSelectStatement<T extends HasResultSet, D> extends Query<T, D>
with SingleTableQueryMixin<T, D>, LimitContainerMixin<T, D>, Selectable<D>
implements BaseSelectStatement {
implements BaseSelectStatement<D> {
/// Whether duplicate rows should be eliminated from the result (this is a
/// `SELECT DISTINCT` statement in sql). Defaults to false.
final bool distinct;
@ -39,7 +41,8 @@ class SimpleSelectStatement<T extends HasResultSet, D> extends Query<T, D>
Set<ResultSetImplementation> get watchedTables => {table};
@override
int get _returnedColumnCount => table.$columns.length;
Iterable<(Expression, String)> get _expandedColumns =>
table.$columns.map((e) => (e, e.name));
@override
String? _nameForColumn(Expression expression) {
@ -54,8 +57,8 @@ class SimpleSelectStatement<T extends HasResultSet, D> extends Query<T, D>
void writeStartPart(GenerationContext ctx) {
ctx.buffer
..write(_beginOfSelect(distinct))
..write(' * FROM ${table.tableWithAlias}');
ctx.watchedTables.add(table);
..write(' * FROM ');
ctx.writeResultSet(table);
}
@override
@ -82,8 +85,13 @@ class SimpleSelectStatement<T extends HasResultSet, D> extends Query<T, D>
});
}
@override
FutureOr<D> _mapRow(Map<String, Object?> row) {
return table.map(row);
}
Future<List<D>> _mapResponse(List<Map<String, Object?>> rows) {
return rows.mapAsyncAndAwait(table.map);
return rows.mapAsyncAndAwait(_mapRow);
}
/// Creates a select statement that operates on more than one table by

View File

@ -31,11 +31,11 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
///
/// Each table column can be uniquely identified by its (potentially aliased)
/// table and its name. So a column named `id` in a table called `users` would
/// be written as `users.id AS "users.id"`. These columns will NOT be written
/// into this map.
/// be written as `users.id AS "users.id"`. These columns are also included in
/// the map when added through [addColumns], but they have a predicatable name.
///
/// Other expressions used as columns will be included here. There just named
/// in increasing order, so something like `AS c3`.
/// More interestingly, other expressions used as columns will be included
/// here. They're just named in increasing order, so something like `AS c3`.
final Map<Expression, String> _columnAliases = {};
/// The tables this select statement reads from
@ -44,13 +44,16 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
Set<ResultSetImplementation> get watchedTables => _queriedTables().toSet();
@override
int get _returnedColumnCount {
return _joins.fold(_selectedColumns.length, (prev, join) {
if (join.includeInResult ?? _includeJoinedTablesInResult) {
return prev + (join.table as ResultSetImplementation).$columns.length;
Iterable<(Expression<Object>, String)> get _expandedColumns sync* {
for (final column in _selectedColumns) {
yield (column, _columnAliases[column]!);
}
for (final table in _queriedTables(true)) {
for (final column in table.$columns) {
yield (column, _nameForTableColumn(column));
}
return prev;
});
}
}
@override
@ -122,9 +125,8 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
chosenAlias = _nameForTableColumn(column,
generatingForView: ctx.generatingForView);
} else {
chosenAlias = 'c$i';
chosenAlias = _columnAliases[column]!;
}
_columnAliases[column] = chosenAlias;
column.writeInto(ctx);
ctx.buffer
@ -133,8 +135,8 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
..write('"');
}
ctx.buffer.write(' FROM ${table.tableWithAlias}');
ctx.watchedTables.add(table);
ctx.buffer.write(' FROM ');
ctx.writeResultSet(table);
if (_joins.isNotEmpty) {
ctx.writeWhitespace();
@ -195,7 +197,21 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
/// - The docs on expressions: https://drift.simonbinder.eu/docs/getting-started/expressions/
/// {@endtemplate}
void addColumns(Iterable<Expression> expressions) {
_selectedColumns.addAll(expressions);
for (final expression in expressions) {
// Otherwise, we generate an alias.
_columnAliases.putIfAbsent(expression, () {
// Only add the column if it hasn't been added yet - it's fine if the
// same column is added multiple times through the Dart API, they will
// read from the same SQL column internally.
_selectedColumns.add(expression);
if (expression is GeneratedColumn) {
return _nameForTableColumn(expression);
} else {
return 'c${_columnAliases.length}';
}
});
}
}
/// Adds more joined tables to this [JoinedSelectStatement].
@ -233,14 +249,14 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
return database
.createStream(fetcher)
.asyncMapPerSubscription((rows) => _mapResponse(ctx, rows));
.asyncMapPerSubscription((rows) => _mapResponse(rows));
}
@override
Future<List<TypedResult>> get() async {
final ctx = constructQuery();
final raw = await _getRaw(ctx);
return _mapResponse(ctx, raw);
return _mapResponse(raw);
}
Future<List<Map<String, Object?>>> _getRaw(GenerationContext ctx) {
@ -260,24 +276,26 @@ class JoinedSelectStatement<FirstT extends HasResultSet, FirstD>
});
}
Future<List<TypedResult>> _mapResponse(
GenerationContext ctx, List<Map<String, Object?>> rows) {
return Future.wait(rows.map((row) async {
final readTables = <ResultSetImplementation, dynamic>{};
@override
Future<TypedResult> _mapRow(Map<String, Object?> row) async {
final readTables = <ResultSetImplementation, dynamic>{};
for (final table in _queriedTables(true)) {
final prefix = '${table.aliasedName}.';
// if all columns of this table are null, skip the table
if (table.$columns.any((c) => row[prefix + c.$name] != null)) {
readTables[table] =
await table.map(row, tablePrefix: table.aliasedName);
}
for (final table in _queriedTables(true)) {
final prefix = '${table.aliasedName}.';
// if all columns of this table are null, skip the table
if (table.$columns.any((c) => row[prefix + c.$name] != null)) {
readTables[table] =
await table.map(row, tablePrefix: table.aliasedName);
}
}
final driftRow = QueryRow(row, database);
return TypedResult(
readTables, driftRow, _LazyExpressionMap(_columnAliases, driftRow));
}));
final driftRow = QueryRow(row, database);
return TypedResult(
readTables, driftRow, _LazyExpressionMap(_columnAliases, driftRow));
}
Future<List<TypedResult>> _mapResponse(List<Map<String, Object?>> rows) {
return Future.wait(rows.map(_mapRow));
}
Never _warnAboutDuplicate(

View File

@ -1,6 +1,6 @@
name: drift
description: Drift is a reactive library to store relational data in Dart and Flutter applications.
version: 2.10.0
version: 2.11.0-dev
repository: https://github.com/simolus3/drift
homepage: https://drift.simonbinder.eu/
issue_tracker: https://github.com/simolus3/drift/issues

View File

@ -224,7 +224,7 @@ void main() {
'c.desc': 'Description',
'c.description_in_upper_case': 'DESCRIPTION',
'c.priority': 1,
'c4': 11
'c0': 11
}
];
});
@ -234,7 +234,7 @@ void main() {
verify(executor.runSelect(
'SELECT "c"."id" AS "c.id", "c"."desc" AS "c.desc", '
'"c"."priority" AS "c.priority", "c"."description_in_upper_case" AS '
'"c.description_in_upper_case", LENGTH("c"."desc") AS "c4" '
'"c.description_in_upper_case", LENGTH("c"."desc") AS "c0" '
'FROM "categories" "c";',
[],
));
@ -273,7 +273,7 @@ void main() {
'c.desc': 'Description',
'c.description_in_upper_case': 'DESCRIPTION',
'c.priority': 1,
'c4': 11,
'c0': 11,
},
];
});
@ -283,7 +283,7 @@ void main() {
verify(executor.runSelect(
'SELECT "c"."id" AS "c.id", "c"."desc" AS "c.desc", "c"."priority" AS "c.priority"'
', "c"."description_in_upper_case" AS "c.description_in_upper_case", '
'LENGTH("c"."desc") AS "c4" '
'LENGTH("c"."desc") AS "c0" '
'FROM "categories" "c" '
'INNER JOIN "todos" "t" ON "c"."id" = "t"."category";',
[],
@ -328,7 +328,7 @@ void main() {
'c.id': 3,
'c.desc': 'desc',
'c.priority': 0,
'c4': 10,
'c0': 10,
'c.description_in_upper_case': 'DESC',
}
];
@ -340,7 +340,7 @@ void main() {
'SELECT "c"."id" AS "c.id", "c"."desc" AS "c.desc", '
'"c"."priority" AS "c.priority", '
'"c"."description_in_upper_case" AS "c.description_in_upper_case", '
'COUNT("t"."id") AS "c4" '
'COUNT("t"."id") AS "c0" '
'FROM "categories" "c" INNER JOIN "todos" "t" ON "t"."category" = "c"."id" '
'GROUP BY "c"."id" HAVING COUNT("t"."id") >= ?;',
[10]));
@ -474,4 +474,72 @@ void main() {
throwsA(isNot(isA<DriftWrappedException>())),
);
});
group('subquery', () {
test('can be joined', () async {
final subquery = Subquery(
db.select(db.todosTable)
..orderBy([(row) => OrderingTerm.desc(row.title.length)])
..limit(10),
's',
);
final query = db.selectOnly(db.categories)
..addColumns([db.categories.id])
..join([
innerJoin(subquery,
subquery.ref(db.todosTable.category).equalsExp(db.categories.id))
]);
await query.get();
verify(
executor.runSelect(
'SELECT "categories"."id" AS "categories.id" FROM "categories" '
'INNER JOIN (SELECT * FROM "todos" '
'ORDER BY LENGTH("todos"."title") DESC LIMIT 10) s '
'ON "s"."category" = "categories"."id";',
argThat(isEmpty),
),
);
});
test('use column from subquery', () async {
when(executor.runSelect(any, any)).thenAnswer((_) {
return Future.value([
{'c0': 42}
]);
});
final sumOfTitleLength = db.todosTable.title.length.sum();
final subquery = Subquery(
db.selectOnly(db.todosTable)
..addColumns([db.todosTable.category, sumOfTitleLength])
..groupBy([db.todosTable.category]),
's');
final readableLength = subquery.ref(sumOfTitleLength);
final query = db.selectOnly(db.categories)
..addColumns([readableLength])
..join([
innerJoin(subquery,
subquery.ref(db.todosTable.category).equalsExp(db.categories.id))
]);
final row = await query.getSingle();
verify(
executor.runSelect(
'SELECT "s"."c1" AS "c0" FROM "categories" '
'INNER JOIN ('
'SELECT "todos"."category" AS "todos.category", '
'SUM(LENGTH("todos"."title")) AS "c1" FROM "todos" '
'GROUP BY "todos"."category") s '
'ON "s"."todos.category" = "categories"."id";',
argThat(isEmpty),
),
);
expect(row.read(readableLength), 42);
});
});
}

View File

@ -214,4 +214,30 @@ void main() {
await pumpEventQueue();
db.markTablesUpdated([db.categories]);
});
test('select from subquery', () async {
final data = [
{
'id': 10,
'title': null,
'content': 'Content',
'category': null,
}
];
when(executor.runSelect(any, any)).thenAnswer((_) => Future.value(data));
final subquery = Subquery(db.todosTable.select(), 's');
final rows = await db.select(subquery).get();
expect(rows, [
TodoEntry(
id: 10,
title: null,
content: 'Content',
category: null,
)
]);
verify(executor.runSelect('SELECT * FROM (SELECT * FROM "todos") s;', []));
});
}

View File

@ -50,4 +50,52 @@ void main() {
expect(await db.users.all().get(), [user]);
});
test('subqueries', () async {
await db.batch((batch) {
batch.insertAll(db.categories, [
CategoriesCompanion.insert(description: 'a'),
CategoriesCompanion.insert(description: 'b'),
]);
batch.insertAll(
db.todosTable,
[
TodosTableCompanion.insert(content: 'aaaaa', category: Value(1)),
TodosTableCompanion.insert(content: 'aa', category: Value(1)),
TodosTableCompanion.insert(content: 'bbbbbb', category: Value(2)),
],
);
});
// Now write a query returning the amount of content chars in each
// category (written using subqueries).
final subqueryContentLength = db.todosTable.content.length.sum();
final subquery = Subquery(
db.selectOnly(db.todosTable)
..addColumns([db.todosTable.category, subqueryContentLength])
..groupBy([db.todosTable.category]),
's');
final readableLength = subquery.ref(subqueryContentLength);
final query = db.selectOnly(db.categories)
..addColumns([db.categories.id, readableLength])
..join([
innerJoin(subquery,
subquery.ref(db.todosTable.category).equalsExp(db.categories.id))
])
..orderBy([OrderingTerm.asc(db.categories.id)]);
final rows = await query.get();
expect(rows, hasLength(2));
final first = rows[0];
final second = rows[1];
expect(first.read(db.categories.id), 1);
expect(first.read(readableLength), 7);
expect(second.read(db.categories.id), 2);
expect(second.read(readableLength), 6);
});
}

View File

@ -145,7 +145,7 @@ class SqlEngine {
/// Parses multiple [sql] statements, separated by a semicolon.
///
/// You can use the [AstNode.children] of the returned [ParseResult.rootNode]
/// You can use the [AstNode.childNodes] of the returned [ParseResult.rootNode]
/// to inspect the returned statements.
ParseResult parseMultiple(String sql) {
final tokens = tokenize(sql);