From 80ced55d329c6da0deb948ffb1d56ed5576c8706 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 25 Jan 2020 21:56:09 +0100 Subject: [PATCH] Improve api for "group by" statements, documentation --- .../expressions.md | 52 +++++++++++++++- .../en/docs/Advanced Features/joins.md | 62 ++++++++++++++++++- moor/lib/src/runtime/api/query_engine.dart | 40 ++++++++++++ .../query_builder/components/join.dart | 36 ++++++++--- .../query_builder/statements/query.dart | 2 +- .../statements/select/select_with_join.dart | 35 +++++++---- moor/test/join_test.dart | 55 +++++++++++++++- 7 files changed, 259 insertions(+), 23 deletions(-) rename docs/content/en/docs/{Getting started => Advanced Features}/expressions.md (65%) diff --git a/docs/content/en/docs/Getting started/expressions.md b/docs/content/en/docs/Advanced Features/expressions.md similarity index 65% rename from docs/content/en/docs/Getting started/expressions.md rename to docs/content/en/docs/Advanced Features/expressions.md index 4bfa5345..78bde563 100644 --- a/docs/content/en/docs/Getting started/expressions.md +++ b/docs/content/en/docs/Advanced Features/expressions.md @@ -1,8 +1,11 @@ --- title: "Expressions" -linkTitle: "Expressions" +linkTitle: "Expressions in Dart" description: Deep-dive into what kind of SQL expressions can be written in Dart weight: 200 + +# used to be in the "getting started" section +url: docs/getting-started/expressions/ --- Expressions are pieces of sql that return a value when the database interprets them. @@ -68,6 +71,8 @@ fields from that date: select(users)..where((u) => u.birthDate.year.isLessThan(1950)) ``` +The individual fileds like `year`, `month` and so on are expressions themselves. This means +that you can use operators and comparisons on them. To obtain the current date or the current time as an expression, use the `currentDate` and `currentDateAndTime` constants provided by moor. @@ -80,6 +85,51 @@ select(animals)..where((a) => a.amountOfLegs.isIn([3, 7, 4, 2]); Again, the `isNotIn` function works the other way around. +## Aggregate functions (like count and sum) {#aggregate} + +Since moor 2.4, [aggregate functions](https://www.sqlite.org/lang_aggfunc.html) are available +from the Dart api. Unlike regular functions, aggregate functions operate on multiple rows at +once. +By default, they combine all rows that would be returned by the select statement into a single value. +You can also make them run over different groups in the result by using +[group by]({{< relref "joins.md#group-by" >}}). + +### Comparing + +You can use the `min` and `max` methods on numeric and datetime expressions. They return the smallest +or largest value in the result set, respectively. + +### Arithmetic + +The `avg`, `sum` and `total` methods are available. For instance, you could watch the average length of +a todo item with this query: +```dart +Stream averageItemLength() { + final avgLength = todos.content.length.avg(); + + final query = selectOnly(todos) + ..addColumns([avgLength]); + + return query.map((row) => row.read(avgLength)).watchSingle(); +} +``` + +__Note__: We're using `selectOnly` instead of `select` because we're not interested in any colum that +`todos` provides - we only care about the average length. More details are available +[here]({{< relref "joins.md#group-by" >}}) + +### Counting + +Sometimes, it's useful to count how many rows are present in a group. By using the +[table layout from the example]({{}}), this +query will report how many todo entries are associated to each category: + +If you don't want to count duplicate values, you can use `count(distinct: true)`. +Sometimes, you only need to count values that match a condition. For that, you can +use the `filter` parameter. + +To count all rows (instead of a single value), you can use the top-level `countAll()`. + ## Custom expressions If you want to inline custom sql into Dart queries, you can use a `CustomExpression` class. It takes a `sql` parameter that let's you write custom expressions: diff --git a/docs/content/en/docs/Advanced Features/joins.md b/docs/content/en/docs/Advanced Features/joins.md index fae3100f..62b8913c 100644 --- a/docs/content/en/docs/Advanced Features/joins.md +++ b/docs/content/en/docs/Advanced Features/joins.md @@ -146,4 +146,64 @@ SELECT FROM routes INNER JOIN geo_points s ON s.id = routes.start INNER JOIN geo_points d ON d.id = routes.destination -``` \ No newline at end of file +``` + +## Group by + +Sometimes, you need to run queries that _aggregate_ data, meaning that data you're interested in +comes from multiple rows. Common questions include + +- how many todo entries are in each category? +- how many entries did a user complete each month? +- what's the average length of a todo entry? + +What these queries have in common is that data from multiple rows needs to be combined into a single +row. In sql, this can be achieved with "aggregate functins", for which moor has +[builtin support]({{< relref "expressions.md#aggregate" >}}). + +_Additional info_: A good tutorial for group by in sql is available [here](https://www.sqlitetutorial.net/sqlite-group-by/). + +To write a query that answers the first question for us, we can use the `count` function. +We're going to select all categories and join each todo entry for each category. What's special is that we set +`useColumns: false` on the join. We do that because we're not interested in the columns of the todo item. +We only care about how many there are. By default, moor would attempt to read each todo item when it appears +in a join. + +```dart +final amountOfTodos = todos.id.count(); + +final query = db.select(categories).join([ + innerJoin( + todos, + todos.category.equalsExp(categories.id), + useColumns: false, + ) +]); +query + ..addColumns([amountOfTodos]) + ..groupBy([categories.id]); + +final result = await query.get(); + +for (final row in result) { + print('there are ${row.read(amountOfTodos)} entries in ${row.readTable(todos)}'); +} +``` + +To find the average length of a todo entry, we use `avg`. In this case, we don't even have to use +a `join` since all the data comes from a single table (todos). +That's a problem though - in the join, we used `useColumns: false` because we weren't interested +in the columns of each todo item. Here we don't care about an individual item either, but there's +no join where we could set that flag. +Moor provides a special method for this case - instead of using `select`, we use `selectOnly`. +The "only" means that moor will only report columns we added via "addColumns". In a regular select, +all columns from the table would be selected, which is what you'd usually need. + +```dart +Stream averageItemLength() { + final avgLength = todos.content.length.avg(); + final query = selectOnly(todos)..addColumns([avgLength]); + + return query.map((row) => row.read(avgLength)).watchSingle(); +} +``` diff --git a/moor/lib/src/runtime/api/query_engine.dart b/moor/lib/src/runtime/api/query_engine.dart index 355d86c0..a6112241 100644 --- a/moor/lib/src/runtime/api/query_engine.dart +++ b/moor/lib/src/runtime/api/query_engine.dart @@ -96,6 +96,46 @@ mixin QueryEngine on DatabaseConnectionUser { distinct: distinct); } + /// Starts a complex statement on [table] that doesn't necessarily use all of + /// [table]'s columns. + /// + /// Unlike [select], which automatically selects all columns of [table], this + /// method is suitable for more advanced queries that can use [table] without + /// using their column. As an example, assuming we have a table `comments` + /// with a `TextColumn content`, this query would report the average length of + /// a comment: + /// ```dart + /// Stream watchAverageCommentLength() { + /// final avgLength = comments.content.length.avg(); + /// final query = selectWithoutResults(comments) + /// ..addColumns([avgLength]); + /// + /// return query.map((row) => row.read(avgLength)).watchSingle(); + /// } + /// ``` + /// + /// While this query reads from `comments`, it doesn't use all of it's columns + /// (in fact, it uses none of them!). This makes it suitable for + /// [selectOnly] instead of [select]. + /// + /// The [distinct] parameter (defaults to false) can be used to remove + /// duplicate rows from the result set. + /// + /// For simple queries, use [select]. + /// + /// See also: + /// - the documentation on [aggregate expressions](https://moor.simonbinder.eu/docs/getting-started/expressions/#aggregate) + /// - the documentation on [group by](https://moor.simonbinder.eu/docs/advanced-features/joins/#group-by) + @protected + @visibleForTesting + JoinedSelectStatement selectOnly( + TableInfo table, { + bool distinct = false, + }) { + return JoinedSelectStatement( + _resolvedEngine, table, const [], distinct, false); + } + /// Starts a [DeleteStatement] that can be used to delete rows from a table. /// /// See the [documentation](https://moor.simonbinder.eu/docs/getting-started/writing_queries/#updates-and-deletes) diff --git a/moor/lib/src/runtime/query_builder/components/join.dart b/moor/lib/src/runtime/query_builder/components/join.dart index 73c30453..af908478 100644 --- a/moor/lib/src/runtime/query_builder/components/join.dart +++ b/moor/lib/src/runtime/query_builder/components/join.dart @@ -33,9 +33,16 @@ class Join extends Component { /// that must be matched for the join. final Expression on; + /// Whether [table] should appear in the result set (defaults to true). + /// + /// It can be useful to exclude some tables. Sometimes, tables are used in a + /// join only to run aggregate functions on them. + final bool includeInResult; + /// Constructs a [Join] by providing the relevant fields. [on] is optional for /// [_JoinType.cross]. - Join._(this.type, this.table, this.on); + Join._(this.type, this.table, this.on, {bool includeInResult}) + : includeInResult = includeInResult ?? true; @override void writeInto(GenerationContext context) { @@ -53,28 +60,43 @@ class Join extends Component { /// Creates a sql inner join that can be used in [SimpleSelectStatement.join]. /// +/// {@template moor_join_include_results} +/// The optional [useColumns] parameter (defaults to true) can be used to +/// exclude the [other] table from the result set. When set, +/// [TypedResult.readTable] will return `null` for that table. +/// {@endtemplate} +/// /// See also: +/// - https://moor.simonbinder.eu/docs/advanced-features/joins/#joins /// - http://www.sqlitetutorial.net/sqlite-inner-join/ Join innerJoin( - TableInfo other, Expression on) { - return Join._(_JoinType.inner, other, on); + TableInfo other, Expression on, + {bool useColumns}) { + return Join._(_JoinType.inner, other, on, includeInResult: useColumns); } /// Creates a sql left outer join that can be used in /// [SimpleSelectStatement.join]. /// +/// {@macro moor_join_include_results} +/// /// See also: +/// - https://moor.simonbinder.eu/docs/advanced-features/joins/#joins /// - http://www.sqlitetutorial.net/sqlite-left-join/ Join leftOuterJoin( - TableInfo other, Expression on) { - return Join._(_JoinType.leftOuter, other, on); + TableInfo other, Expression on, + {bool useColumns}) { + return Join._(_JoinType.leftOuter, other, on, includeInResult: useColumns); } /// Creates a sql cross join that can be used in /// [SimpleSelectStatement.join]. /// +/// {@macro moor_join_include_results} +/// /// See also: +/// - https://moor.simonbinder.eu/docs/advanced-features/joins/#joins /// - http://www.sqlitetutorial.net/sqlite-cross-join/ -Join crossJoin(TableInfo other) { - return Join._(_JoinType.cross, other, null); +Join crossJoin(TableInfo other, {bool useColumns}) { + return Join._(_JoinType.cross, other, null, includeInResult: useColumns); } diff --git a/moor/lib/src/runtime/query_builder/statements/query.dart b/moor/lib/src/runtime/query_builder/statements/query.dart index 47cc053c..4477f759 100644 --- a/moor/lib/src/runtime/query_builder/statements/query.dart +++ b/moor/lib/src/runtime/query_builder/statements/query.dart @@ -56,9 +56,9 @@ abstract class Query { needsWhitespace = true; writeWithSpace(whereExpr); + writeWithSpace(_groupBy); writeWithSpace(orderByExpr); writeWithSpace(limitExpr); - writeWithSpace(_groupBy); ctx.buffer.write(';'); diff --git a/moor/lib/src/runtime/query_builder/statements/select/select_with_join.dart b/moor/lib/src/runtime/query_builder/statements/select/select_with_join.dart index bb77a2a7..93a641bb 100644 --- a/moor/lib/src/runtime/query_builder/statements/select/select_with_join.dart +++ b/moor/lib/src/runtime/query_builder/statements/select/select_with_join.dart @@ -11,20 +11,21 @@ class JoinedSelectStatement /// instead. JoinedSelectStatement( QueryEngine database, TableInfo table, this._joins, - [this.distinct = false]) + [this.distinct = false, this._includeMainTableInResult = true]) : super(database, table) { // use all columns across all tables as result column for this query - _selectedColumns ??= - _tables.expand((t) => t.$columns).cast().toList(); + _selectedColumns.addAll( + _queriedTables(true).expand((t) => t.$columns).cast()); } /// Whether to generate a `SELECT DISTINCT` query that will remove duplicate /// rows from the result set. final bool distinct; + final bool _includeMainTableInResult; final List _joins; /// All columns that we're selecting from. - List _selectedColumns; + final List _selectedColumns = []; /// The `AS` aliases generated for each column that isn't from a table. /// @@ -39,11 +40,23 @@ class JoinedSelectStatement /// The tables this select statement reads from @visibleForOverriding - Set get watchedTables => _tables.toSet(); + Set get watchedTables => _queriedTables().toSet(); - // fixed order to make testing easier - Iterable get _tables => - [table].followedBy(_joins.map((j) => j.table)); + /// Lists all tables this query reads from. + /// + /// If [onlyResults] (defaults to false) is set, only tables that are included + /// in the result set are returned. + Iterable _queriedTables([bool onlyResults = false]) sync* { + if (!onlyResults || _includeMainTableInResult) { + yield table; + } + + for (final join in _joins) { + if (onlyResults && !join.includeInResult) continue; + + yield join.table; + } + } @override void writeStartPart(GenerationContext ctx) { @@ -164,7 +177,7 @@ class JoinedSelectStatement return await e.runSelect(ctx.sql, ctx.boundVariables); } catch (e, s) { final foundTables = {}; - for (final table in _tables) { + for (final table in _queriedTables()) { if (!foundTables.add(table.$tableName)) { _warnAboutDuplicate(e, s, table); } @@ -174,13 +187,11 @@ class JoinedSelectStatement } }); - final tables = _tables; - return results.map((row) { final readTables = {}; final readColumns = {}; - for (final table in tables) { + for (final table in _queriedTables(true)) { final prefix = '${table.$tableName}.'; // if all columns of this table are null, skip the table if (table.$columns.any((c) => row[prefix + c.$name] != null)) { diff --git a/moor/test/join_test.dart b/moor/test/join_test.dart index 43782d20..7faf9578 100644 --- a/moor/test/join_test.dart +++ b/moor/test/join_test.dart @@ -1,4 +1,4 @@ -import 'package:moor/moor.dart'; +import 'package:moor/moor.dart' hide isNull; import 'package:test/test.dart'; import 'data/tables/todos.dart'; import 'data/utils/mocks.dart'; @@ -172,6 +172,59 @@ void main() { expect(result.read(descriptionLength), 11); }); + test('group by', () async { + final categories = db.alias(db.categories, 'c'); + final todos = db.alias(db.todosTable, 't'); + final amountOfTodos = todos.id.count(); + + final query = db.select(categories).join([ + innerJoin( + todos, + todos.category.equalsExp(categories.id), + useColumns: false, + ) + ]); + query + ..addColumns([amountOfTodos]) + ..groupBy([categories.id]); + + when(executor.runSelect(any, any)).thenAnswer((_) async { + return [ + {'c.id': 3, 'c.desc': 'desc', 'c2': 10} + ]; + }); + + final result = await query.getSingle(); + + verify(executor.runSelect( + 'SELECT c.id AS "c.id", c.`desc` AS "c.desc", COUNT(t.id) AS "c2" ' + 'FROM categories c INNER JOIN todos t ON t.category = c.id ' + 'GROUP BY c.id;', + [])); + + expect(result.readTable(todos), isNull); + expect(result.readTable(categories), Category(id: 3, description: 'desc')); + expect(result.read(amountOfTodos), 10); + }); + + test('selectWithoutResults', () async { + final avgLength = db.todosTable.content.length.avg(); + final query = db.selectOnly(db.todosTable)..addColumns([avgLength]); + + when(executor.runSelect(any, any)).thenAnswer((_) async { + return [ + {'c0': 3.0} + ]; + }); + + final result = await query.map((row) => row.read(avgLength)).getSingle(); + + verify(executor.runSelect( + 'SELECT AVG(LENGTH(todos.content)) AS "c0" FROM todos;', [])); + + expect(result, 3.0); + }); + test('injects custom error message when a table is used multiple times', () async { when(executor.runSelect(any, any)).thenAnswer((_) => Future.error('nah'));