From e7fd94e9db06b962d1a9ad2644fbd9274d8866b5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 20 Jul 2019 11:00:51 +0200 Subject: [PATCH 001/117] Copy-paste ffi sqlite library from putraxor/sqlite3_ffi --- sqlite/.gitignore | 7 + sqlite/README.md | 41 ++ sqlite/analysis_options.yaml | 1 + sqlite/docs/android.md | 53 +++ sqlite/docs/lib/scenario-default.svg | 130 ++++++ sqlite/docs/lib/scenario-full.svg | 149 +++++++ sqlite/docs/sqlite-tutorial.md | 234 ++++++++++ sqlite/example/main.dart | 89 ++++ sqlite/lib/sqlite.dart | 10 + sqlite/lib/src/bindings/bindings.dart | 411 ++++++++++++++++++ sqlite/lib/src/bindings/constants.dart | 182 ++++++++ sqlite/lib/src/bindings/signatures.dart | 62 +++ sqlite/lib/src/bindings/types.dart | 77 ++++ .../src/collections/closable_iterator.dart | 29 ++ sqlite/lib/src/database.dart | 326 ++++++++++++++ sqlite/lib/src/ffi/arena.dart | 57 +++ sqlite/lib/src/ffi/cstring.dart | 43 ++ sqlite/lib/src/ffi/dylib_utils.dart | 20 + sqlite/pubspec.yaml | 9 + sqlite/test/sqlite_test.dart | 164 +++++++ 20 files changed, 2094 insertions(+) create mode 100644 sqlite/.gitignore create mode 100644 sqlite/README.md create mode 100644 sqlite/analysis_options.yaml create mode 100644 sqlite/docs/android.md create mode 100644 sqlite/docs/lib/scenario-default.svg create mode 100644 sqlite/docs/lib/scenario-full.svg create mode 100644 sqlite/docs/sqlite-tutorial.md create mode 100644 sqlite/example/main.dart create mode 100644 sqlite/lib/sqlite.dart create mode 100644 sqlite/lib/src/bindings/bindings.dart create mode 100644 sqlite/lib/src/bindings/constants.dart create mode 100644 sqlite/lib/src/bindings/signatures.dart create mode 100644 sqlite/lib/src/bindings/types.dart create mode 100644 sqlite/lib/src/collections/closable_iterator.dart create mode 100644 sqlite/lib/src/database.dart create mode 100644 sqlite/lib/src/ffi/arena.dart create mode 100644 sqlite/lib/src/ffi/cstring.dart create mode 100644 sqlite/lib/src/ffi/dylib_utils.dart create mode 100644 sqlite/pubspec.yaml create mode 100644 sqlite/test/sqlite_test.dart diff --git a/sqlite/.gitignore b/sqlite/.gitignore new file mode 100644 index 00000000..7a6bc2ee --- /dev/null +++ b/sqlite/.gitignore @@ -0,0 +1,7 @@ +.dart_tool +.gdb_history +.packages +.vscode +pubspec.lock +test.db +test.db-journal \ No newline at end of file diff --git a/sqlite/README.md b/sqlite/README.md new file mode 100644 index 00000000..58542d02 --- /dev/null +++ b/sqlite/README.md @@ -0,0 +1,41 @@ +# SQLite3 wrapper for dart:ffi + +This is an illustrative sample for how to use `dart:ffi`. + + +## Building and Running this Sample + +Building and running this sample is done through pub. +Running `pub get` and `pub run example/main` should produce the following output. + +```sh +$ pub get +Resolving dependencies... (6.8s) ++ analyzer 0.35.4 +... ++ yaml 2.1.15 +Downloading analyzer 0.35.4... +Downloading kernel 0.3.14... +Downloading front_end 0.1.14... +Changed 47 dependencies! +Precompiling executables... (18.0s) +Precompiled test:test. + +``` + +``` +$ pub run example/main +1 Chocolade chip cookie Chocolade cookie foo +2 Ginger cookie null 42 +3 Cinnamon roll null null +1 Chocolade chip cookie Chocolade cookie foo +2 Ginger cookie null 42 +expected exception on accessing result data after close: The result has already been closed. +expected this query to fail: no such column: non_existing_column (Code 1: SQL logic error) +``` + +## Tutorial + +A tutorial walking through the code is available in [docs/sqlite-tutorial.md](docs/sqlite-tutorial.md). +For information on how to use this package within a Flutter app, see [docs/android.md]. +(Note: iOS is not yet supported). diff --git a/sqlite/analysis_options.yaml b/sqlite/analysis_options.yaml new file mode 100644 index 00000000..ca56bebe --- /dev/null +++ b/sqlite/analysis_options.yaml @@ -0,0 +1 @@ +# Using an empty analysis options so that this package gets analyzed with the less strict default rules \ No newline at end of file diff --git a/sqlite/docs/android.md b/sqlite/docs/android.md new file mode 100644 index 00000000..07f3e47d --- /dev/null +++ b/sqlite/docs/android.md @@ -0,0 +1,53 @@ +**This documentation is for demonstration/testing purposes only!** + +# Using FFI with Flutter + +## Android + +Before using the FFI on Android, you need to procure an Android-compatible build of the native library you want to link against. +It's important that the shared object(s) be compatible with ABI version you wish to target (or else, that you have multiple builds for different ABIs). +See [https://developer.android.com/ndk/guides/abis] for more details on Android ABIs. +Within Flutter, the target ABI is controlled by the `--target-platform` parameter to the `flutter` command. + +The workflow for packaging a native library will depend significantly on the library itself, but to illustrate the challenges at play, we'll demonstrate how to build the SQLite library from source to use with the FFI on an Android device. + +### Building SQLite for Android + +Every Android device ships with a copy of the SQLite library (`/system/lib/libsqlite.so`). +Unfortunately, this library cannot be loaded directly by apps (see [https://developer.android.com/about/versions/nougat/android-7.0-changes#ndk]). +It is accessible only through Java. +Instead, we can build SQLite directly with the NDK. + +First, download the SQLite "amalgamation" source from [https://www.sqlite.org/download.html]. +For the sake of brevity, we'll assume the file has been saved as `sqlite-amalgamation-XXXXXXX.zip`, the Android SDK (with NDK extension) is available in `~/Android`, and we're on a Linux workstation. + +```sh +unzip sqlite-amalgamation-XXXXXXX.zip +cd sqlite-amalgamation-XXXXXXX +~/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang -c sqlite3.c -o sqlite3.o +~/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android-ld -shared sqlite3.o -o libsqlite3.so +``` + +Note the use of the `aarch64` prefix to the compiler: this indicates that we're building a shared object for the `arm64-v8a` ABI. +This will be important later. + +### Update Gradle script + +Next we need to instruct Gradle to package this library with the app, so it will be available to load off the Android device at runtime. +Create a folder `native-libraries` in the root folder of the app, and update the `android/app/build.gradle` file: + +```groovy +android { + // ... + sourceSets { + main { + jniLibs.srcDir '${project.projectDir.path}/../../native-libraries' + } + } +} +``` + +Within the `native-libraries` folder, the libraries are organized by ABI. +Therefore, we must copy the compiled `libsqlite3.so` into `native-libraries/arm64-v8a/libsqlite3.so`. +If multiple sub-directories are present, the libraries from the sub-directory corresponding to the target ABI will be available in the application's linking path, so the library can be loaded with `ffi.DynamicLibrary.open("libsqlite3.so")` in Dart. +Finally, pass `--target-platform=android-arm64` to the `flutter` command when running or building the app since `libsqlite3.so` was compiled for the `arm64-v8a` ABI. diff --git a/sqlite/docs/lib/scenario-default.svg b/sqlite/docs/lib/scenario-default.svg new file mode 100644 index 00000000..6ffa8a34 --- /dev/null +++ b/sqlite/docs/lib/scenario-default.svg @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.9.4 + 2019-03-13 09:56:08 +0000 + + + Canvas 1 + + + Layer 1 + + + + + Flutter + App + (Imports + package) + + + + + + + Native + Library + + + + + + + dart:ffi + + + + + + + Package + API + (Does not + expose + dart:ffi) + + + + + + + Dart + + + + + + + C / C++ + + + + + + + App + Developer + + + + + + + Package + Developer + + + + + + + Dart + VM + Team + + + + + + + Bindings + + + + + + + Native + Library + Developer + + + + + + + Package + Implementation + (Code which + converts C++ + abstractions into + Dart + abstractions) + + + + + diff --git a/sqlite/docs/lib/scenario-full.svg b/sqlite/docs/lib/scenario-full.svg new file mode 100644 index 00000000..4ae18c50 --- /dev/null +++ b/sqlite/docs/lib/scenario-full.svg @@ -0,0 +1,149 @@ + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.9.4 + 2019-03-13 09:53:08 +0000 + + + Canvas 1 + + + Layer 1 + + + + + Flutter + App + (Imports + package) + + + + + + + Native + Library + + + + + + + dart:ffi + + + + + + + Package + API + (Does not + expose + dart:ffi) + + + + + + + Dart + + + + + + + C / C++ + + + + + + + App + Developer + + + + + + + Package + Developer + + + + + + + Dart + VM + Team + + + + + + + Bindings + + + + + + + Native + Library + Developer + + + + + + + Package + Implementation + (Code which + converts C++ + abstractions into + Dart + abstractions) + + + + + + + Package + Developer + + + + + + + Glue code + (Code which + takes care of + things such as + C++ exceptions) + + + + + diff --git a/sqlite/docs/sqlite-tutorial.md b/sqlite/docs/sqlite-tutorial.md new file mode 100644 index 00000000..c38170c9 --- /dev/null +++ b/sqlite/docs/sqlite-tutorial.md @@ -0,0 +1,234 @@ +# dart:ffi SQLite mini tutorial + +In this mini tutorial we learn how to bind SQLite, a native library, in Dart using Dart's new foreign function interface `dart:ffi`. +We build a package which provides a Dartlike SQLite API using objects and `Iterator`s. +Inside the package we write Dart code which directly invokes C functions and manipulates C memory. + +## Binding C Functions to Dart + +The first step is to load a Native Library: + +```dart +import "dart:ffi"; + +DynamicLibrary sqlite = dlopenPlatformSpecific("sqlite3"); +``` + +In a `DynamicLibrary` we can `lookup` functions. +Let's lookup the function `sqlite3_prepare_v2` in the SQLite library. +That function has the following signature in the library header file. + +```c++ +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +``` + +In order to lookup a function, we need a _C signature_ and a _Dart signature_. + +```dart +typedef sqlite3_prepare_v2_native_t = Int32 Function( + DatabasePointer database, + CString query, + Int32 nbytes, + Pointer statementOut, + Pointer tail); + +typedef Sqlite3_prepare_v2_t = int Function( + DatabasePointer database, + CString query, + int nbytes, + Pointer statementOut, + Pointer tail); +``` + +With these two signatures we can `lookup` the C function and expose it as a Dart function with `asFunction`. + +```dart +Sqlite3_prepare_v2_t sqlite3_prepare_v2 = sqlite + .lookup>("sqlite3_prepare_v2") + .asFunction(); +``` + +Browse the code: [platform specific dynamic library loading](../lib/src/ffi/dylib_utils.dart), [C signatures](../lib/src/bindings/signatures.dart), [Dart signatures and bindings](../lib/src/bindings/bindings.dart), and [dart:ffi dynamic library interface](../../../../sdk/lib/ffi/dynamic_library.dart). + +## Managing C Memory + +In order to call `sqlite3_prepare_v2` to prepare a SQLite statement before executing, we need to be able to pass C pointers to C functions. + +Database and Statement pointers are opaque pointers in the SQLite C API. +We specify these as classes extending `Pointer`. + +```dart +class DatabasePointer extends Pointer {} +class StatementPointer extends Pointer {} +``` + +Strings in C are pointers to character arrays. + +```dart +class CString extends Pointer {} +``` + +Pointers to C integers, floats, an doubles can be read from and written through to `dart:ffi`. +However, before we can write to C memory from dart, we need to `allocate` some memory. + +```dart +Pointer p = allocate(); // Infers type argument allocate(), and allocates 1 byte. +p.store(123); // Stores a Dart int into this C int8. +int v = p.load(); // Infers type argument p.load(), and loads a value from C memory. +``` + +Note that you can only load a Dart `int` from a C `Uint8`. +Trying to load a Dart `double` will result in a runtime exception. + +We've almost modeled C Strings. +The last thing we need is to use this `Pointer` as an array. +We can do this by using `elementAt`. + +```dart +CString string = allocate(count: 4).cast(); // Allocates 4 bytes and casts it to a string. +string.store(73); // Stores 'F' at index 0. +string.elementAt(1).store(73); // Stores 'F' at index 1. +string.elementAt(2).store(70); // Stores 'I' at index 2. +string.elementAt(3).store(0); // Null terminates the string. +``` + +We wrap the above logic of allocating strings in the constructor `CString.allocate`. + +Now we have all ingredients to call `sqlite3_prepare_v2`. + +```dart +Pointer statementOut = allocate(); +CString queryC = CString.allocate(query); +int resultCode = sqlite3_prepare_v2( + _database, queryC, -1, statementOut, fromAddress(0)); +``` + +With `dart:ffi` we are responsible for freeing C memory that we allocate. +So after calling `sqlite3_prepare_v2` we read out the statement pointer, and free the statement pointer pointer and `CString` which held the query string. + +``` +StatementPointer statement = statementOut.load(); +statementOut.free(); +queryC.free(); +``` + +Browse the code: [CString class](../lib/src/ffi/cstring.dart), [code calling sqlite3_prepare_v2](../lib/src/database.dart#57), and [dart:ffi pointer interface](../../../../sdk/lib/ffi/ffi.dart). + +## Dart API + +We would like to present the users of our package with an object oriented API - not exposing any `dart:ffi` objects to them. + +The SQLite C API returns a cursor to the first row of a result after executing a query. +We can read out the columns of this row and move the cursor to the next row. +The most natural way to expose this in Dart is through an `Iterable`. +We provide our package users with the following API. + +```dart +class Result implements Iterable {} + +class Row { + dynamic readColumnByIndex(int columnIndex) {} + dynamic readColumn(String columnName) {} +} +``` + +However, this interface does not completely match the semantics of the C API. +When we start reading the next `Row`, we do no longer have access to the previous `Row`. +We can model this by letting a `Row` keep track if its current or not. + +```dart +class Row { + bool _isCurrentRow = true; + + dynamic readColumnByIndex(int columnIndex) { + if (!_isCurrentRow) { + throw Exception( + "This row is not the current row, reading data from the non-current" + " row is not supported by sqlite."); + } + // ... + } +} +``` + +A second mismatch between Dart and C is that in C we have to manually release resources. +After executing a query and reading its results we need to call `sqlite3_finalize(statement)`. + +We can take two approaches here, either we structure the API in such a way that users of our package (implicitly) release resources, or we use finalizers to release resources. +In this tutorial we take the first approach. + +If our users iterate over all `Row`s, we can implicitly finalize the statement after they are done with the last row. +However, if they decide they do not want to iterate over the whole result, they need to explicitly state this. +In this tutorial, we use the `ClosableIterator` abstraction for `Iterators` with backing resources that need to be `close`d. + +```dart +Result result = d.query(""" + select id, name + from Cookies + ;"""); +for (Row r in result) { + String name = r.readColumn("name"); + print(name); +} +// Implicitly closes the iterator. + +result = d.query(""" + select id, name + from Cookies + ;"""); +for (Row r in result) { + int id = r.readColumn("id"); + if (id == 1) { + result.close(); // Explicitly closes the iterator, releasing underlying resources. + break; + } +} +``` + +Browse the code: [Database, Result, Row](../lib/src/database.dart), and [CloseableIterator](../lib/src/collections/closable_iterator.dart). + +## Architecture Overview + +The following diagram summarized what we have implemented as _package developers_ in this tutorial. + +![architecture](lib/scenario-default.svg) + +As the package developers wrapping an existing native library, we have only written Dart code - not any C/C++ code. +We specified bindings to the native library. +We have provided our package users with an object oriented API without exposing any `dart:ffi` objects. +And finally, we have implemented the package API by calling the C API. + +## Current dart:ffi Development Status + +In this minitutorial we used these `dart:ffi` features: + +* Loading dynamic libararies and looking up C functions in these dynamic libraries. +* Calling C functions, with `dart:ffi` automatically marshalling arguments and return value. +* Manipulating C memory through `Pointer`s with `allocate`, `free`, `load`, `store`, and `elementAt`. + +Features which we did not use in this tutorial: + +* `@struct` on subtypes of `Pointer` to define a struct with fields. (However, this feature is likely to change in the future.) + +Features which `dart:ffi` does not support yet: + +* Callbacks from C back into Dart. +* Finalizers +* C++ Exceptions (Not on roadmap yet.) + +Platform limitations: + +* `dart:ffi` is only enabled on 64 bit Windows, Linux, and MacOS. (Arm64 and 32 bit Intel are under review.) +* `dart:ffi` only works in JIT mode, not in AOT. + +It is possible to work around some of the current limitations by adding a C/C++ layer. +For example we could catch C++ exceptions in a C++ layer, and rethrow them in Dart. +The architecture diagram would change to the following in that case. + +![architecture2](lib/scenario-full.svg) \ No newline at end of file diff --git a/sqlite/example/main.dart b/sqlite/example/main.dart new file mode 100644 index 00000000..436a2830 --- /dev/null +++ b/sqlite/example/main.dart @@ -0,0 +1,89 @@ +import "../lib/sqlite.dart"; + +// ignore_for_file: dead_code + +void main() { + Database d = Database("test.db"); + d.execute("update Cookies set name='Changed' where name=?", + params: ['Google']); + var rows = d.query('select * from Cookies'); + rows.forEach((row) { + var id = row.readColumnByIndexAsInt(0); + var name = row.readColumnByIndexAsText(1); + print('$id $name '); + }); + return; + d.execute("drop table if exists Cookies;"); + d.execute(""" + create table Cookies ( + id integer primary key, + name text not null, + alternative_name text + );"""); + d.execute(""" + insert into Cookies (id, name, alternative_name) + values + (1,'Chocolade chip cookie', 'Chocolade cookie'), + (2,'Ginger cookie', null), + (3,'Cinnamon roll', null) + ;"""); + Result result = d.query(""" + select + id, + name, + alternative_name, + case + when id=1 then 'foo' + when id=2 then 42 + when id=3 then null + end as multi_typed_column + from Cookies + ;"""); + for (Row r in result) { + int id = r.readColumnAsInt("id"); + String name = r.readColumnByIndex(1); + String alternativeName = r.readColumn("alternative_name"); + dynamic multiTypedValue = r.readColumn("multi_typed_column"); + print("$id $name $alternativeName $multiTypedValue"); + } + result = d.query(""" + select + id, + name, + alternative_name, + case + when id=1 then 'foo' + when id=2 then 42 + when id=3 then null + end as multi_typed_column + from Cookies + ;"""); + for (Row r in result) { + int id = r.readColumnAsInt("id"); + String name = r.readColumnByIndex(1); + String alternativeName = r.readColumn("alternative_name"); + dynamic multiTypedValue = r.readColumn("multi_typed_column"); + print("$id $name $alternativeName $multiTypedValue"); + if (id == 2) { + result.close(); + break; + } + } + try { + result.iterator.moveNext(); + } on SQLiteException catch (e) { + print("expected exception on accessing result data after close: $e"); + } + try { + d.query(""" + select + id, + non_existing_column + from Cookies + ;"""); + } on SQLiteException catch (e) { + print("expected this query to fail: $e"); + } + //d.execute("drop table Cookies;"); + d.close(); +} diff --git a/sqlite/lib/sqlite.dart b/sqlite/lib/sqlite.dart new file mode 100644 index 00000000..d6bac5be --- /dev/null +++ b/sqlite/lib/sqlite.dart @@ -0,0 +1,10 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// A synchronous SQLite wrapper. +/// +/// Written using dart:ffi. +library sqlite; + +export "src/database.dart"; diff --git a/sqlite/lib/src/bindings/bindings.dart b/sqlite/lib/src/bindings/bindings.dart new file mode 100644 index 00000000..0b15e10e --- /dev/null +++ b/sqlite/lib/src/bindings/bindings.dart @@ -0,0 +1,411 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:ffi"; + +import "../ffi/cstring.dart"; +import "../ffi/dylib_utils.dart"; + +import "signatures.dart"; +import "types.dart"; + +class _SQLiteBindings { + DynamicLibrary sqlite; + + /// Opening A New Database Connection + /// + /// ^These routines open an SQLite database file as specified by the + /// filename argument. ^The filename argument is interpreted as UTF-8 for + /// sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte + /// order for sqlite3_open16(). ^(A database connection handle is usually + /// returned in *ppDb, even if an error occurs. The only exception is that + /// if SQLite is unable to allocate memory to hold the sqlite3 object, + /// a NULL will be written into *ppDb instead of a pointer to the sqlite3 + /// object.)^ ^(If the database is opened (and/or created) successfully, then + /// [SQLITE_OK] is returned. Otherwise an error code is returned.)^ ^The + /// [sqlite3_errmsg] or sqlite3_errmsg16() routines can be used to obtain + /// an English language description of the error following a failure of any + /// of the sqlite3_open() routines. + int Function(CString filename, Pointer databaseOut, + int flags, CString vfs) sqlite3_open_v2; + + int Function(DatabasePointer database) sqlite3_close_v2; + + /// Compiling An SQL Statement + /// + /// To execute an SQL query, it must first be compiled into a byte-code + /// program using one of these routines. + /// + /// The first argument, "db", is a database connection obtained from a + /// prior successful call to sqlite3_open, [sqlite3_open_v2] or + /// sqlite3_open16. The database connection must not have been closed. + /// + /// The second argument, "zSql", is the statement to be compiled, encoded + /// as either UTF-8 or UTF-16. The sqlite3_prepare() and sqlite3_prepare_v2() + /// interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2() + /// use UTF-16. + /// + /// ^If the nByte argument is less than zero, then zSql is read up to the + /// first zero terminator. ^If nByte is non-negative, then it is the maximum + /// number of bytes read from zSql. ^When nByte is non-negative, the + /// zSql string ends at either the first '\000' or '\u0000' character or + /// the nByte-th byte, whichever comes first. If the caller knows + /// that the supplied string is nul-terminated, then there is a small + /// performance advantage to be gained by passing an nByte parameter that + /// is equal to the number of bytes in the input string including + /// the nul-terminator bytes. + /// + /// ^If pzTail is not NULL then *pzTail is made to point to the first byte + /// past the end of the first SQL statement in zSql. These routines only + /// compile the first statement in zSql, so *pzTail is left pointing to + /// what remains uncompiled. + /// + /// ^*ppStmt is left pointing to a compiled prepared statement that can be + /// executed using sqlite3_step. ^If there is an error, *ppStmt is set + /// to NULL. ^If the input text contains no SQL (if the input is an empty + /// string or a comment) then *ppStmt is set to NULL. + /// The calling procedure is responsible for deleting the compiled + /// SQL statement using [sqlite3_finalize] after it has finished with it. + /// ppStmt may not be NULL. + /// + /// ^On success, the sqlite3_prepare family of routines return [SQLITE_OK]; + /// otherwise an error code is returned. + /// + /// The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are + /// recommended for all new programs. The two older interfaces are retained + /// for backwards compatibility, but their use is discouraged. + /// ^In the "v2" interfaces, the prepared statement + /// that is returned (the sqlite3_stmt object) contains a copy of the + /// original SQL text. This causes the [sqlite3_step] interface to + /// behave differently in three ways: + int Function( + DatabasePointer database, + CString query, + int nbytes, + Pointer statementOut, + Pointer tail) sqlite3_prepare_v2; + + /// Evaluate An SQL Statement + /// + /// After a prepared statement has been prepared using either + /// [sqlite3_prepare_v2] or sqlite3_prepare16_v2() or one of the legacy + /// interfaces sqlite3_prepare() or sqlite3_prepare16(), this function + /// must be called one or more times to evaluate the statement. + /// + /// The details of the behavior of the sqlite3_step() interface depend + /// on whether the statement was prepared using the newer "v2" interface + /// [sqlite3_prepare_v2] and sqlite3_prepare16_v2() or the older legacy + /// interface sqlite3_prepare() and sqlite3_prepare16(). The use of the + /// new "v2" interface is recommended for new applications but the legacy + /// interface will continue to be supported. + /// + /// ^In the legacy interface, the return value will be either [SQLITE_BUSY], + /// [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. + /// ^With the "v2" interface, any of the other [result codes] or + /// [extended result codes] might be returned as well. + /// + /// ^[SQLITE_BUSY] means that the database engine was unable to acquire the + /// database locks it needs to do its job. ^If the statement is a [COMMIT] + /// or occurs outside of an explicit transaction, then you can retry the + /// statement. If the statement is not a [COMMIT] and occurs within an + /// explicit transaction then you should rollback the transaction before + /// continuing. + /// + /// ^[SQLITE_DONE] means that the statement has finished executing + /// successfully. sqlite3_step() should not be called again on this virtual + /// machine without first calling [sqlite3_reset()] to reset the virtual + /// machine back to its initial state. + /// + /// ^If the SQL statement being executed returns any data, then [SQLITE_ROW] + /// is returned each time a new row of data is ready for processing by the + /// caller. The values may be accessed using the [column access functions]. + /// sqlite3_step() is called again to retrieve the next row of data. + /// + /// ^[SQLITE_ERROR] means that a run-time error (such as a constraint + /// violation) has occurred. sqlite3_step() should not be called again on + /// the VM. More information may be found by calling [sqlite3_errmsg()]. + /// ^With the legacy interface, a more specific error code (for example, + /// [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) + /// can be obtained by calling [sqlite3_reset()] on the + /// prepared statement. ^In the "v2" interface, + /// the more specific error code is returned directly by sqlite3_step(). + /// + /// [SQLITE_MISUSE] means that the this routine was called inappropriately. + /// Perhaps it was called on a prepared statement that has + /// already been [sqlite3_finalize | finalized] or on one that had + /// previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could + /// be the case that the same database connection is being used by two or + /// more threads at the same moment in time. + /// + /// For all versions of SQLite up to and including 3.6.23.1, a call to + /// [sqlite3_reset] was required after sqlite3_step() returned anything + /// other than [Errors.SQLITE_ROW] before any subsequent invocation of + /// sqlite3_step(). Failure to reset the prepared statement using + /// [sqlite3_reset()] would result in an [Errors.SQLITE_MISUSE] return from + /// sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began + /// calling [sqlite3_reset] automatically in this circumstance rather + /// than returning [Errors.SQLITE_MISUSE]. This is not considered a + /// compatibility break because any application that ever receives an + /// [Errors.SQLITE_MISUSE] error is broken by definition. The + /// [SQLITE_OMIT_AUTORESET] compile-time option + /// can be used to restore the legacy behavior. + /// + /// Goofy Interface Alert: In the legacy interface, the sqlite3_step() + /// API always returns a generic error code, [SQLITE_ERROR], following any + /// error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call + /// [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the + /// specific [error codes] that better describes the error. + /// We admit that this is a goofy design. The problem has been fixed + /// with the "v2" interface. If you prepare all of your SQL statements + /// using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead + /// of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, + /// then the more specific [error codes] are returned directly + /// by sqlite3_step(). The use of the "v2" interface is recommended. + int Function(StatementPointer statement) sqlite3_step; + + /// CAPI3REF: Reset A Prepared Statement Object + /// + /// The sqlite3_reset() function is called to reset a prepared statement + /// object back to its initial state, ready to be re-executed. + /// ^Any SQL statement variables that had values bound to them using + /// the sqlite3_bind_blob | sqlite3_bind_*() API retain their values. + /// Use sqlite3_clear_bindings() to reset the bindings. + /// + /// ^The [sqlite3_reset] interface resets the prepared statement S + /// back to the beginning of its program. + /// + /// ^If the most recent call to [sqlite3_step] for the + /// prepared statement S returned [Errors.SQLITE_ROW] or [Errors.SQLITE_DONE], + /// or if [sqlite3_step] has never before been called on S, + /// then [sqlite3_reset] returns [Errors.SQLITE_OK]. + /// + /// ^If the most recent call to [sqlite3_step(S)] for the + /// prepared statement S indicated an error, then + /// [sqlite3_reset] returns an appropriate [Errors]. + /// + /// ^The [sqlite3_reset] interface does not change the values + int Function(StatementPointer statement) sqlite3_reset; + + /// Destroy A Prepared Statement Object + /// + /// ^The sqlite3_finalize() function is called to delete a prepared statement. + /// ^If the most recent evaluation of the statement encountered no errors + /// or if the statement is never been evaluated, then sqlite3_finalize() + /// returns SQLITE_OK. ^If the most recent evaluation of statement S failed, + /// then sqlite3_finalize(S) returns the appropriate error code or extended + /// error code. + /// + /// ^The sqlite3_finalize(S) routine can be called at any point during + /// the life cycle of prepared statement S: + /// before statement S is ever evaluated, after + /// one or more calls to [sqlite3_reset], or after any call + /// to [sqlite3_step] regardless of whether or not the statement has + /// completed execution. + /// + /// ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. + /// + /// The application must finalize every prepared statement in order to avoid + /// resource leaks. It is a grievous error for the application to try to use + /// a prepared statement after it has been finalized. Any use of a prepared + /// statement after it has been finalized can result in undefined and + /// undesirable behavior such as segfaults and heap corruption. + int Function(StatementPointer statement) sqlite3_finalize; + + /// Number Of Columns In A Result Set + /// + /// ^Return the number of columns in the result set returned by the + /// prepared statement. ^This routine returns 0 if pStmt is an SQL + /// statement that does not return data (for example an [UPDATE]). + int Function(StatementPointer statement) sqlite3_column_count; + + /// Column Names In A Result Set + /// + /// ^These routines return the name assigned to a particular column + /// in the result set of a SELECT statement. ^The sqlite3_column_name() + /// interface returns a pointer to a zero-terminated UTF-8 string + /// and sqlite3_column_name16() returns a pointer to a zero-terminated + /// UTF-16 string. ^The first parameter is the prepared statement + /// that implements the SELECT statement. ^The second parameter is the + /// column number. ^The leftmost column is number 0. + /// + /// ^The returned string pointer is valid until either the prepared statement + /// is destroyed by [sqlite3_finalize] or until the statement is automatically + /// reprepared by the first call to [sqlite3_step] for a particular run + /// or until the next call to + /// sqlite3_column_name() or sqlite3_column_name16() on the same column. + /// + /// ^If sqlite3_malloc() fails during the processing of either routine + /// (for example during a conversion from UTF-8 to UTF-16) then a + /// NULL pointer is returned. + /// + /// ^The name of a result column is the value of the "AS" clause for + /// that column, if there is an AS clause. If there is no AS clause + /// then the name of the column is unspecified and may change from + CString Function(StatementPointer statement, int columnIndex) + sqlite3_column_name; + + /// CAPI3REF: Declared Datatype Of A Query Result + /// + /// ^(The first parameter is a prepared statement. + /// If this statement is a SELECT statement and the Nth column of the + /// returned result set of that SELECT is a table column (not an + /// expression or subquery) then the declared type of the table + /// column is returned.)^ ^If the Nth column of the result set is an + /// expression or subquery, then a NULL pointer is returned. + /// ^The returned string is always UTF-8 encoded. + /// + /// ^(For example, given the database schema: + /// + /// CREATE TABLE t1(c1 VARIANT); + /// + /// and the following statement to be compiled: + /// + /// SELECT c1 + 1, c1 FROM t1; + /// + /// this routine would return the string "VARIANT" for the second result + /// column (i==1), and a NULL pointer for the first result column (i==0).)^ + /// + /// ^SQLite uses dynamic run-time typing. ^So just because a column + /// is declared to contain a particular type does not mean that the + /// data stored in that column is of the declared type. SQLite is + /// strongly typed, but the typing is dynamic not static. ^Type + /// is associated with individual values, not with the containers + /// used to hold those values. + CString Function(StatementPointer statement, int columnIndex) + sqlite3_column_decltype; + + int Function(StatementPointer statement, int columnIndex) sqlite3_column_type; + + ValuePointer Function(StatementPointer statement, int columnIndex) + sqlite3_column_value; + + double Function(StatementPointer statement, int columnIndex) + sqlite3_column_double; + + int Function(StatementPointer statement, int columnIndex) sqlite3_column_int; + + CString Function(StatementPointer statement, int columnIndex) + sqlite3_column_text; + + /// The sqlite3_errstr() interface returns the English-language text that + /// describes the result code, as UTF-8. Memory to hold the error message + /// string is managed internally and must not be freed by the application. + CString Function(int code) sqlite3_errstr; + + /// Error Codes And Messages + /// + /// ^The sqlite3_errcode() interface returns the numeric [result code] or + /// [extended result code] for the most recent failed sqlite3_* API call + /// associated with a [database connection]. If a prior API call failed + /// but the most recent API call succeeded, the return value from + /// sqlite3_errcode() is undefined. ^The sqlite3_extended_errcode() + /// interface is the same except that it always returns the + /// [extended result code] even when extended result codes are + /// disabled. + /// + /// ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language + /// text that describes the error, as either UTF-8 or UTF-16 respectively. + /// ^(Memory to hold the error message string is managed internally. + /// The application does not need to worry about freeing the result. + /// However, the error string might be overwritten or deallocated by + /// subsequent calls to other SQLite interface functions.)^ + /// + /// When the serialized [threading mode] is in use, it might be the + /// case that a second error occurs on a separate thread in between + /// the time of the first error and the call to these interfaces. + /// When that happens, the second error will be reported since these + /// interfaces always report the most recent result. To avoid + /// this, each thread can obtain exclusive use of the [database connection] D + /// by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning + /// to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after + /// all calls to the interfaces listed here are completed. + /// + /// If an interface fails with SQLITE_MISUSE, that means the interface + /// was invoked incorrectly by the application. In that case, the + /// error code and message may or may not be set. + CString Function(DatabasePointer database) sqlite3_errmsg; + + int Function(StatementPointer statement, int columnIndex, double value) + sqlite3_bind_double; + int Function(StatementPointer statement, int columnIndex, int value) + sqlite3_bind_int; + int Function(StatementPointer statement, int columnIndex, CString value) + sqlite3_bind_text; + + _SQLiteBindings() { + sqlite = dlopenPlatformSpecific("sqlite3"); + + sqlite3_bind_double = sqlite + .lookup>( + "sqlite3_bind_double") + .asFunction(); + sqlite3_bind_int = sqlite + .lookup>("sqlite3_bind_int") + .asFunction(); + sqlite3_bind_text = sqlite + .lookup>("sqlite3_bind_text") + .asFunction(); + + sqlite3_open_v2 = sqlite + .lookup>("sqlite3_open_v2") + .asFunction(); + sqlite3_close_v2 = sqlite + .lookup>("sqlite3_close_v2") + .asFunction(); + sqlite3_prepare_v2 = sqlite + .lookup>( + "sqlite3_prepare_v2") + .asFunction(); + sqlite3_step = sqlite + .lookup>("sqlite3_step") + .asFunction(); + sqlite3_reset = sqlite + .lookup>("sqlite3_reset") + .asFunction(); + sqlite3_finalize = sqlite + .lookup>("sqlite3_finalize") + .asFunction(); + sqlite3_errstr = sqlite + .lookup>("sqlite3_errstr") + .asFunction(); + sqlite3_errmsg = sqlite + .lookup>("sqlite3_errmsg") + .asFunction(); + sqlite3_column_count = sqlite + .lookup>( + "sqlite3_column_count") + .asFunction(); + sqlite3_column_name = sqlite + .lookup>( + "sqlite3_column_name") + .asFunction(); + sqlite3_column_decltype = sqlite + .lookup>( + "sqlite3_column_decltype") + .asFunction(); + sqlite3_column_type = sqlite + .lookup>( + "sqlite3_column_type") + .asFunction(); + sqlite3_column_value = sqlite + .lookup>( + "sqlite3_column_value") + .asFunction(); + sqlite3_column_double = sqlite + .lookup>( + "sqlite3_column_double") + .asFunction(); + sqlite3_column_int = sqlite + .lookup>( + "sqlite3_column_int") + .asFunction(); + sqlite3_column_text = sqlite + .lookup>( + "sqlite3_column_text") + .asFunction(); + } +} + +_SQLiteBindings _cachedBindings; +_SQLiteBindings get bindings => _cachedBindings ??= _SQLiteBindings(); diff --git a/sqlite/lib/src/bindings/constants.dart b/sqlite/lib/src/bindings/constants.dart new file mode 100644 index 00000000..71aa82ed --- /dev/null +++ b/sqlite/lib/src/bindings/constants.dart @@ -0,0 +1,182 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Result Codes +/// +/// Many SQLite functions return an integer result code from the set shown +/// here in order to indicates success or failure. +/// +/// New error codes may be added in future versions of SQLite. +/// +/// See also: SQLITE_IOERR_READ | extended result codes, +/// sqlite3_vtab_on_conflict() SQLITE_ROLLBACK | result codes. +class Errors { + /// Successful result + static const int SQLITE_OK = 0; + + /// Generic error + static const int SQLITE_ERROR = 1; + + /// Internal logic error in SQLite + static const int SQLITE_INTERNAL = 2; + + /// Access permission denied + static const int SQLITE_PERM = 3; + + /// Callback routine requested an abort + static const int SQLITE_ABORT = 4; + + /// The database file is locked + static const int SQLITE_BUSY = 5; + + /// A table in the database is locked + static const int SQLITE_LOCKED = 6; + + /// A malloc() failed + static const int SQLITE_NOMEM = 7; + + /// Attempt to write a readonly database + static const int SQLITE_READONLY = 8; + + /// Operation terminated by sqlite3_interrupt() + static const int SQLITE_INTERRUPT = 9; + + /// Some kind of disk I/O error occurred + static const int SQLITE_IOERR = 10; + + /// The database disk image is malformed + static const int SQLITE_CORRUPT = 11; + + /// Unknown opcode in sqlite3_file_control() + static const int SQLITE_NOTFOUND = 12; + + /// Insertion failed because database is full + static const int SQLITE_FULL = 13; + + /// Unable to open the database file + static const int SQLITE_CANTOPEN = 14; + + /// Database lock protocol error + static const int SQLITE_PROTOCOL = 15; + + /// Internal use only + static const int SQLITE_EMPTY = 16; + + /// The database schema changed + static const int SQLITE_SCHEMA = 17; + + /// String or BLOB exceeds size limit + static const int SQLITE_TOOBIG = 18; + + /// Abort due to constraint violation + static const int SQLITE_CONSTRAINT = 19; + + /// Data type mismatch + static const int SQLITE_MISMATCH = 20; + + /// Library used incorrectly + static const int SQLITE_MISUSE = 21; + + /// Uses OS features not supported on host + static const int SQLITE_NOLFS = 22; + + /// Authorization denied + static const int SQLITE_AUTH = 23; + + /// Not used + static const int SQLITE_FORMAT = 24; + + /// 2nd parameter to sqlite3_bind out of range + static const int SQLITE_RANGE = 25; + + /// File opened that is not a database file + static const int SQLITE_NOTADB = 26; + + /// Notifications from sqlite3_log() + static const int SQLITE_NOTICE = 27; + + /// Warnings from sqlite3_log() + static const int SQLITE_WARNING = 28; + + /// sqlite3_step() has another row ready + static const int SQLITE_ROW = 100; + + /// sqlite3_step() has finished executing + static const int SQLITE_DONE = 101; +} + +/// Flags For File Open Operations +/// +/// These bit values are intended for use in the +/// 3rd parameter to the [sqlite3_open_v2()] interface and +/// in the 4th parameter to the [sqlite3_vfs.xOpen] method. +class Flags { + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_READONLY = 0x00000001; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_READWRITE = 0x00000002; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_CREATE = 0x00000004; + + /// VFS only + static const int SQLITE_OPEN_DELETEONCLOSE = 0x00000008; + + /// VFS only + static const int SQLITE_OPEN_EXCLUSIVE = 0x00000010; + + /// VFS only + static const int SQLITE_OPEN_AUTOPROXY = 0x00000020; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_URI = 0x00000040; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_MEMORY = 0x00000080; + + /// VFS only + static const int SQLITE_OPEN_MAIN_DB = 0x00000100; + + /// VFS only + static const int SQLITE_OPEN_TEMP_DB = 0x00000200; + + /// VFS only + static const int SQLITE_OPEN_TRANSIENT_DB = 0x00000400; + + /// VFS only + static const int SQLITE_OPEN_MAIN_JOURNAL = 0x00000800; + + /// VFS only + static const int SQLITE_OPEN_TEMP_JOURNAL = 0x00001000; + + /// VFS only + static const int SQLITE_OPEN_SUBJOURNAL = 0x00002000; + + /// VFS only + static const int SQLITE_OPEN_MASTER_JOURNAL = 0x00004000; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_NOMUTEX = 0x00008000; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_FULLMUTEX = 0x00010000; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_SHAREDCACHE = 0x00020000; + + /// Ok for sqlite3_open_v2() + static const int SQLITE_OPEN_PRIVATECACHE = 0x00040000; + + /// VFS only + static const int SQLITE_OPEN_WAL = 0x00080000; +} + +class Types { + static const int SQLITE_INTEGER = 1; + static const int SQLITE_FLOAT = 2; + static const int SQLITE_TEXT = 3; + static const int SQLITE_BLOB = 4; + static const int SQLITE_NULL = 5; +} diff --git a/sqlite/lib/src/bindings/signatures.dart b/sqlite/lib/src/bindings/signatures.dart new file mode 100644 index 00000000..59b0face --- /dev/null +++ b/sqlite/lib/src/bindings/signatures.dart @@ -0,0 +1,62 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:ffi"; + +import "../ffi/cstring.dart"; + +import "types.dart"; + +typedef sqlite3_open_v2_native_t = Int32 Function( + CString filename, Pointer ppDb, Int32 flags, CString vfs); + +typedef sqlite3_close_v2_native_t = Int32 Function(DatabasePointer database); + +typedef sqlite3_prepare_v2_native_t = Int32 Function( + DatabasePointer database, + CString query, + Int32 nbytes, + Pointer statementOut, + Pointer tail); + +typedef sqlite3_step_native_t = Int32 Function(StatementPointer statement); + +typedef sqlite3_reset_native_t = Int32 Function(StatementPointer statement); + +typedef sqlite3_finalize_native_t = Int32 Function(StatementPointer statement); + +typedef sqlite3_errstr_native_t = CString Function(Int32 error); + +typedef sqlite3_errmsg_native_t = CString Function(DatabasePointer database); + +typedef sqlite3_column_count_native_t = Int32 Function( + StatementPointer statement); + +typedef sqlite3_column_name_native_t = CString Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_decltype_native_t = CString Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_type_native_t = Int32 Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_value_native_t = ValuePointer Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_double_native_t = Double Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_int_native_t = Int32 Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_text_native_t = CString Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_bind_double_native = Int32 Function( + StatementPointer statement, Int32 columnIndex, Double value); +typedef sqlite3_bind_int_native = Int32 Function( + StatementPointer statement, Int32 columnIndex, Int32 value); +typedef sqlite3_bind_text_native = Int32 Function( + StatementPointer statement, Int32 columnIndex, CString value); diff --git a/sqlite/lib/src/bindings/types.dart b/sqlite/lib/src/bindings/types.dart new file mode 100644 index 00000000..95035f74 --- /dev/null +++ b/sqlite/lib/src/bindings/types.dart @@ -0,0 +1,77 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:ffi"; + +class FunctionPointer extends Pointer {} + +/// Database Connection Handle +/// +/// Each open SQLite database is represented by a pointer to an instance of +/// the opaque structure named "sqlite3". It is useful to think of an sqlite3 +/// pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +/// [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +/// is its destructor. There are many other interfaces (such as +/// [sqlite3_prepare_v2()], [sqlite3_create_function()], and +/// [sqlite3_busy_timeout()] to name but three) that are methods on an +class DatabasePointer extends Pointer {} + +/// SQL Statement Object +/// +/// An instance of this object represents a single SQL statement. +/// This object is variously known as a "prepared statement" or a +/// "compiled SQL statement" or simply as a "statement". +/// +/// The life of a statement object goes something like this: +/// +///
    +///
  1. Create the object using [sqlite3_prepare_v2()] or a related +/// function. +///
  2. Bind values to [host parameters] using the sqlite3_bind_*() +/// interfaces. +///
  3. Run the SQL by calling [sqlite3_step()] one or more times. +///
  4. Reset the statement using [sqlite3_reset()] then go back +/// to step 2. Do this zero or more times. +///
  5. Destroy the object using [sqlite3_finalize()]. +///
+/// +/// Refer to documentation on individual methods above for additional +/// information. +class StatementPointer extends Pointer {} + +/// Dynamically Typed Value Object +/// +/// SQLite uses the sqlite3_value object to represent all values +/// that can be stored in a database table. SQLite uses dynamic typing +/// for the values it stores. ^Values stored in sqlite3_value objects +/// can be integers, floating point values, strings, BLOBs, or NULL. +/// +/// An sqlite3_value object may be either "protected" or "unprotected". +/// Some interfaces require a protected sqlite3_value. Other interfaces +/// will accept either a protected or an unprotected sqlite3_value. +/// Every interface that accepts sqlite3_value arguments specifies +/// whether or not it requires a protected sqlite3_value. +/// +/// The terms "protected" and "unprotected" refer to whether or not +/// a mutex is held. An internal mutex is held for a protected +/// sqlite3_value object but no mutex is held for an unprotected +/// sqlite3_value object. If SQLite is compiled to be single-threaded +/// (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) +/// or if SQLite is run in one of reduced mutex modes +/// [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] +/// then there is no distinction between protected and unprotected +/// sqlite3_value objects and they can be used interchangeably. However, +/// for maximum code portability it is recommended that applications +/// still make the distinction between protected and unprotected +/// sqlite3_value objects even when not strictly required. +/// +/// ^The sqlite3_value objects that are passed as parameters into the +/// implementation of [application-defined SQL functions] are protected. +/// ^The sqlite3_value object returned by +/// [sqlite3_column_value()] is unprotected. +/// Unprotected sqlite3_value objects may only be used with +/// [sqlite3_result_value()] and [sqlite3_bind_value()]. +/// The [sqlite3_value_blob | sqlite3_value_type()] family of +/// interfaces require protected sqlite3_value objects. +class ValuePointer extends Pointer {} diff --git a/sqlite/lib/src/collections/closable_iterator.dart b/sqlite/lib/src/collections/closable_iterator.dart new file mode 100644 index 00000000..a86a58b2 --- /dev/null +++ b/sqlite/lib/src/collections/closable_iterator.dart @@ -0,0 +1,29 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// This iterator should be [close]d after use. +/// +/// [ClosableIterator]s often use resources which should be freed after use. +/// The consumer of the iterator can either manually [close] the iterator, or +/// consume all elements on which the iterator will automatically be closed. +abstract class ClosableIterator extends Iterator { + /// Close this iterator. + void close(); + + /// Moves to the next element and [close]s the iterator if it was the last + /// element. + bool moveNext(); +} + +/// This iterable's iterator should be [close]d after use. +/// +/// Companion class of [ClosableIterator]. +abstract class ClosableIterable extends Iterable { + /// Close this iterables iterator. + void close(); + + /// Returns a [ClosableIterator] that allows iterating the elements of this + /// [ClosableIterable]. + ClosableIterator get iterator; +} diff --git a/sqlite/lib/src/database.dart b/sqlite/lib/src/database.dart new file mode 100644 index 00000000..c25b5cd8 --- /dev/null +++ b/sqlite/lib/src/database.dart @@ -0,0 +1,326 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:collection"; +import 'dart:convert'; +import "dart:ffi"; + +import "bindings/bindings.dart"; +import "bindings/types.dart"; +import "bindings/constants.dart"; +import "collections/closable_iterator.dart"; +import "ffi/cstring.dart"; + +/// [Database] represents an open connection to a SQLite database. +/// +/// All functions against a database may throw [SQLiteError]. +/// +/// This database interacts with SQLite synchonously. +class Database { + DatabasePointer _database; + bool _open = false; + + /// Open a database located at the file [path]. + Database(String path, + [int flags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE]) { + Pointer dbOut = allocate(); + CString pathC = CString.allocate(path); + final int resultCode = + bindings.sqlite3_open_v2(pathC, dbOut, flags, fromAddress(0)); + _database = dbOut.load(); + dbOut.free(); + pathC.free(); + + if (resultCode == Errors.SQLITE_OK) { + _open = true; + } else { + // Even if "open" fails, sqlite3 will still create a database object. We + // can just destroy it. + SQLiteException exception = _loadError(resultCode); + close(); + throw exception; + } + } + + /// Close the database. + /// + /// This should only be called once on a database unless an exception is + /// thrown. It should be called at least once to finalize the database and + /// avoid resource leaks. + void close() { + assert(_open); + final int resultCode = bindings.sqlite3_close_v2(_database); + if (resultCode == Errors.SQLITE_OK) { + _open = false; + } else { + throw _loadError(resultCode); + } + } + + /// Execute a query, discarding any returned rows. + void execute(String query, {List params}) { + Pointer statementOut = allocate(); + CString queryC = CString.allocate(query); + int resultCode = bindings.sqlite3_prepare_v2( + _database, queryC, -1, statementOut, fromAddress(0)); + + StatementPointer statement = statementOut.load(); + if (params != null) { + for (var i = 0; i < params.length; i++) { + if (params[i].runtimeType.toString() == 'int') { + bindings.sqlite3_bind_int(statement, i + 1, params[i]); + } + if (params[i].runtimeType.toString() == 'double') { + bindings.sqlite3_bind_double(statement, i + 1, params[i]); + } + if (params[i].runtimeType.toString() == 'String') { + String param = utf8.decode(utf8.encode(params[i])); + var text = CString.allocate(param); + bindings.sqlite3_bind_text(statement, i + 1, text); + text.free(); + } + } + } + statementOut.free(); + queryC.free(); + + while (resultCode == Errors.SQLITE_ROW || resultCode == Errors.SQLITE_OK) { + resultCode = bindings.sqlite3_step(statement); + } + bindings.sqlite3_finalize(statement); + if (resultCode != Errors.SQLITE_DONE) { + throw _loadError(resultCode); + } + } + + /// Evaluate a query and return the resulting rows as an iterable. + Result query(String query) { + Pointer statementOut = allocate(); + CString queryC = CString.allocate(query); + int resultCode = bindings.sqlite3_prepare_v2( + _database, queryC, -1, statementOut, fromAddress(0)); + StatementPointer statement = statementOut.load(); + + statementOut.free(); + queryC.free(); + + if (resultCode != Errors.SQLITE_OK) { + bindings.sqlite3_finalize(statement); + throw _loadError(resultCode); + } + + Map columnIndices = {}; + int columnCount = bindings.sqlite3_column_count(statement); + for (int i = 0; i < columnCount; i++) { + String columnName = + CString.fromUtf8(bindings.sqlite3_column_name(statement, i)); + columnIndices[columnName] = i; + } + + return Result._(this, statement, columnIndices); + } + + SQLiteException _loadError([int errorCode]) { + String errorMessage = CString.fromUtf8(bindings.sqlite3_errmsg(_database)); + if (errorCode == null) { + return SQLiteException(errorMessage); + } + String errorCodeExplanation = + CString.fromUtf8(bindings.sqlite3_errstr(errorCode)); + return SQLiteException( + "$errorMessage (Code $errorCode: $errorCodeExplanation)"); + } +} + +/// [Result] represents a [Database.query]'s result and provides an [Iterable] +/// interface for the results to be consumed. +/// +/// Please note that this iterator should be [close]d manually if not all [Row]s +/// are consumed. +class Result extends IterableBase implements ClosableIterable { + final Database _database; + final ClosableIterator _iterator; + final StatementPointer _statement; + final Map _columnIndices; + + Row _currentRow = null; + + Result._( + this._database, + this._statement, + this._columnIndices, + ) : _iterator = _ResultIterator(_statement, _columnIndices) {} + + void close() => _iterator.close(); + + ClosableIterator get iterator => _iterator; +} + +class _ResultIterator implements ClosableIterator { + final StatementPointer _statement; + final Map _columnIndices; + + Row _currentRow = null; + bool _closed = false; + + _ResultIterator(this._statement, this._columnIndices) {} + + bool moveNext() { + if (_closed) { + throw SQLiteException("The result has already been closed."); + } + _currentRow?._setNotCurrent(); + int stepResult = bindings.sqlite3_step(_statement); + if (stepResult == Errors.SQLITE_ROW) { + _currentRow = Row._(_statement, _columnIndices); + return true; + } else { + close(); + return false; + } + } + + Row get current { + if (_closed) { + throw SQLiteException("The result has already been closed."); + } + return _currentRow; + } + + void close() { + _currentRow?._setNotCurrent(); + _closed = true; + bindings.sqlite3_finalize(_statement); + } +} + +class Row { + final StatementPointer _statement; + final Map _columnIndices; + + bool _isCurrentRow = true; + + Row._(this._statement, this._columnIndices) {} + + /// Reads column [columnName]. + /// + /// By default it returns a dynamically typed value. If [convert] is set to + /// [Convert.StaticType] the value is converted to the static type computed + /// for the column by the query compiler. + dynamic readColumn(String columnName, + {Convert convert = Convert.DynamicType}) { + return readColumnByIndex(_columnIndices[columnName], convert: convert); + } + + /// Reads column [columnName]. + /// + /// By default it returns a dynamically typed value. If [convert] is set to + /// [Convert.StaticType] the value is converted to the static type computed + /// for the column by the query compiler. + dynamic readColumnByIndex(int columnIndex, + {Convert convert = Convert.DynamicType}) { + _checkIsCurrentRow(); + + Type dynamicType; + if (convert == Convert.DynamicType) { + dynamicType = + _typeFromCode(bindings.sqlite3_column_type(_statement, columnIndex)); + } else { + dynamicType = _typeFromText(CString.fromUtf8( + bindings.sqlite3_column_decltype(_statement, columnIndex))); + } + + switch (dynamicType) { + case Type.Integer: + return readColumnByIndexAsInt(columnIndex); + case Type.Text: + return readColumnByIndexAsText(columnIndex); + case Type.Null: + return null; + break; + default: + } + } + + /// Reads column [columnName] and converts to [Type.Integer] if not an + /// integer. + int readColumnAsInt(String columnName) { + return readColumnByIndexAsInt(_columnIndices[columnName]); + } + + /// Reads column [columnIndex] and converts to [Type.Integer] if not an + /// integer. + int readColumnByIndexAsInt(int columnIndex) { + _checkIsCurrentRow(); + return bindings.sqlite3_column_int(_statement, columnIndex); + } + + /// Reads column [columnName] and converts to [Type.Text] if not text. + String readColumnAsText(String columnName) { + return readColumnByIndexAsText(_columnIndices[columnName]); + } + + /// Reads column [columnIndex] and converts to [Type.Text] if not text. + String readColumnByIndexAsText(int columnIndex) { + _checkIsCurrentRow(); + return CString.fromUtf8( + bindings.sqlite3_column_text(_statement, columnIndex)); + } + + void _checkIsCurrentRow() { + if (!_isCurrentRow) { + throw Exception( + "This row is not the current row, reading data from the non-current" + " row is not supported by sqlite."); + } + } + + void _setNotCurrent() { + _isCurrentRow = false; + } +} + +Type _typeFromCode(int code) { + switch (code) { + case Types.SQLITE_INTEGER: + return Type.Integer; + case Types.SQLITE_FLOAT: + return Type.Float; + case Types.SQLITE_TEXT: + return Type.Text; + case Types.SQLITE_BLOB: + return Type.Blob; + case Types.SQLITE_NULL: + return Type.Null; + } + throw Exception("Unknown type [$code]"); +} + +Type _typeFromText(String textRepresentation) { + switch (textRepresentation) { + case "integer": + return Type.Integer; + case "float": + return Type.Float; + case "text": + return Type.Text; + case "blob": + return Type.Blob; + case "null": + return Type.Null; + } + if (textRepresentation == null) return Type.Null; + throw Exception("Unknown type [$textRepresentation]"); +} + +enum Type { Integer, Float, Text, Blob, Null } + +enum Convert { DynamicType, StaticType } + +class SQLiteException { + final String message; + SQLiteException(this.message); + + String toString() => message; +} diff --git a/sqlite/lib/src/ffi/arena.dart b/sqlite/lib/src/ffi/arena.dart new file mode 100644 index 00000000..2e19d55f --- /dev/null +++ b/sqlite/lib/src/ffi/arena.dart @@ -0,0 +1,57 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:async"; +import "dart:ffi"; + +/// [Arena] manages allocated C memory. +/// +/// Arenas are zoned. +class Arena { + Arena(); + + List> _allocations = []; + + /// Bound the lifetime of [ptr] to this [Arena]. + T scoped(T ptr) { + _allocations.add(ptr.cast()); + return ptr; + } + + /// Frees all memory pointed to by [Pointer]s in this arena. + void finalize() { + for (final ptr in _allocations) { + ptr.free(); + } + } + + /// The last [Arena] in the zone. + factory Arena.current() { + return Zone.current[#_currentArena]; + } +} + +/// Bound the lifetime of [ptr] to the current [Arena]. +T scoped(T ptr) => Arena.current().scoped(ptr); + +class RethrownError { + dynamic original; + StackTrace originalStackTrace; + RethrownError(this.original, this.originalStackTrace); + toString() => """RethrownError(${original}) +${originalStackTrace}"""; +} + +/// Runs the [body] in an [Arena] freeing all memory which is [scoped] during +/// execution of [body] at the end of the execution. +R runArena(R Function(Arena) body) { + Arena arena = Arena(); + try { + return runZoned(() => body(arena), + zoneValues: {#_currentArena: arena}, + onError: (error, st) => throw RethrownError(error, st)); + } finally { + arena.finalize(); + } +} diff --git a/sqlite/lib/src/ffi/cstring.dart b/sqlite/lib/src/ffi/cstring.dart new file mode 100644 index 00000000..da095e47 --- /dev/null +++ b/sqlite/lib/src/ffi/cstring.dart @@ -0,0 +1,43 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import "dart:convert"; +import "dart:ffi"; + +import "arena.dart"; + +/// Represents a String in C memory, managed by an [Arena]. +class CString extends Pointer { + /// Allocates a [CString] in the current [Arena] and populates it with + /// [dartStr]. + factory CString(String dartStr) => CString.inArena(Arena.current(), dartStr); + + /// Allocates a [CString] in [arena] and populates it with [dartStr]. + factory CString.inArena(Arena arena, String dartStr) => + arena.scoped(CString.allocate(dartStr)); + + /// Allocate a [CString] not managed in and populates it with [dartStr]. + /// + /// This [CString] is not managed by an [Arena]. Please ensure to [free] the + /// memory manually! + factory CString.allocate(String dartStr) { + List units = Utf8Encoder().convert(dartStr); + Pointer str = allocate(count: units.length + 1); + for (int i = 0; i < units.length; ++i) { + str.elementAt(i).store(units[i]); + } + str.elementAt(units.length).store(0); + return str.cast(); + } + + /// Read the string for C memory into Dart. + static String fromUtf8(CString str) { + if (str == null) return null; + int len = 0; + while (str.elementAt(++len).load() != 0); + List units = List(len); + for (int i = 0; i < len; ++i) units[i] = str.elementAt(i).load(); + return Utf8Decoder().convert(units); + } +} diff --git a/sqlite/lib/src/ffi/dylib_utils.dart b/sqlite/lib/src/ffi/dylib_utils.dart new file mode 100644 index 00000000..fb8153af --- /dev/null +++ b/sqlite/lib/src/ffi/dylib_utils.dart @@ -0,0 +1,20 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi' as ffi; +import 'dart:io' show Platform; + +String _platformPath(String name, {String path}) { + if (path == null) path = ""; + if (Platform.isLinux || Platform.isAndroid) + return path + "lib" + name + ".so"; + if (Platform.isMacOS) return path + "lib" + name + ".dylib"; + if (Platform.isWindows) return path + name + ".dll"; + throw Exception("Platform not implemented"); +} + +ffi.DynamicLibrary dlopenPlatformSpecific(String name, {String path}) { + String fullPath = _platformPath(name, path: path); + return ffi.DynamicLibrary.open(fullPath); +} diff --git a/sqlite/pubspec.yaml b/sqlite/pubspec.yaml new file mode 100644 index 00000000..efcf9d41 --- /dev/null +++ b/sqlite/pubspec.yaml @@ -0,0 +1,9 @@ +name: sqlite3_ffi +version: 0.0.1 +description: >- + Sqlite3 wrapper for dart:ffi. +author: Daco Harkes , Samir Jindel +environment: + sdk: '>=2.1.0 <3.0.0' +dev_dependencies: + test: ^1.5.3 \ No newline at end of file diff --git a/sqlite/test/sqlite_test.dart b/sqlite/test/sqlite_test.dart new file mode 100644 index 00000000..95bbc6c1 --- /dev/null +++ b/sqlite/test/sqlite_test.dart @@ -0,0 +1,164 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// VMOptions=--optimization-counter-threshold=5 + +import "package:test/test.dart"; + +import '../lib/sqlite.dart'; + +void main() { + test("sqlite integration test", () { + Database d = Database("test.db"); + d.execute("drop table if exists Cookies;"); + d.execute(""" + create table Cookies ( + id integer primary key, + name text not null, + alternative_name text + );"""); + d.execute(""" + insert into Cookies (id, name, alternative_name) + values + (1,'Chocolade chip cookie', 'Chocolade cookie'), + (2,'Ginger cookie', null), + (3,'Cinnamon roll', null) + ;"""); + Result result = d.query(""" + select + id, + name, + alternative_name, + case + when id=1 then 'foo' + when id=2 then 42 + when id=3 then null + end as multi_typed_column + from Cookies + ;"""); + for (Row r in result) { + int id = r.readColumnAsInt("id"); + expect(true, 1 <= id && id <= 3); + String name = r.readColumnByIndex(1); + expect(true, name is String); + String alternativeName = r.readColumn("alternative_name"); + expect(true, alternativeName is String || alternativeName == null); + dynamic multiTypedValue = r.readColumn("multi_typed_column"); + expect( + true, + multiTypedValue == 42 || + multiTypedValue == 'foo' || + multiTypedValue == null); + print("$id $name $alternativeName $multiTypedValue"); + } + result = d.query(""" + select + id, + name, + alternative_name, + case + when id=1 then 'foo' + when id=2 then 42 + when id=3 then null + end as multi_typed_column + from Cookies + ;"""); + for (Row r in result) { + int id = r.readColumnAsInt("id"); + expect(true, 1 <= id && id <= 3); + String name = r.readColumnByIndex(1); + expect(true, name is String); + String alternativeName = r.readColumn("alternative_name"); + expect(true, alternativeName is String || alternativeName == null); + dynamic multiTypedValue = r.readColumn("multi_typed_column"); + expect( + true, + multiTypedValue == 42 || + multiTypedValue == 'foo' || + multiTypedValue == null); + print("$id $name $alternativeName $multiTypedValue"); + if (id == 2) { + result.close(); + break; + } + } + try { + result.iterator.moveNext(); + } on SQLiteException catch (e) { + print("expected exception on accessing result data after close: $e"); + } + try { + d.query(""" + select + id, + non_existing_column + from Cookies + ;"""); + } on SQLiteException catch (e) { + print("expected this query to fail: $e"); + } + d.execute("drop table Cookies;"); + d.close(); + }); + + test("concurrent db open and queries", () { + Database d = Database("test.db"); + Database d2 = Database("test.db"); + d.execute("drop table if exists Cookies;"); + d.execute(""" + create table Cookies ( + id integer primary key, + name text not null, + alternative_name text + );"""); + d.execute(""" + insert into Cookies (id, name, alternative_name) + values + (1,'Chocolade chip cookie', 'Chocolade cookie'), + (2,'Ginger cookie', null), + (3,'Cinnamon roll', null) + ;"""); + Result r = d.query("select * from Cookies;"); + Result r2 = d2.query("select * from Cookies;"); + r.iterator..moveNext(); + r2.iterator..moveNext(); + r.iterator..moveNext(); + Result r3 = d2.query("select * from Cookies;"); + r3.iterator..moveNext(); + expect(2, r.iterator.current.readColumn("id")); + expect(1, r2.iterator.current.readColumn("id")); + expect(1, r3.iterator.current.readColumn("id")); + r.close(); + r2.close(); + r3.close(); + d.close(); + d2.close(); + }); + + test("stress test", () { + Database d = Database("test.db"); + d.execute("drop table if exists Cookies;"); + d.execute(""" + create table Cookies ( + id integer primary key, + name text not null, + alternative_name text + );"""); + int repeats = 100; + for (int i = 0; i < repeats; i++) { + d.execute(""" + insert into Cookies (name, alternative_name) + values + ('Chocolade chip cookie', 'Chocolade cookie'), + ('Ginger cookie', null), + ('Cinnamon roll', null) + ;"""); + } + Result r = d.query("select count(*) from Cookies;"); + int count = r.first.readColumnByIndexAsInt(0); + expect(count, 3 * repeats); + r.close(); + d.close(); + }); +} From 5dde29301535e8eae4ebc4e82c93330a97c00848 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 20 Jul 2019 11:56:56 +0200 Subject: [PATCH 002/117] Start with primitive VM implementation --- moor/lib/moor_vm.dart | 12 ++++ moor/lib/src/vm/vm_database.dart | 81 +++++++++++++++++++++++++ moor/pubspec.yaml | 3 + sqlite/README.md | 6 ++ sqlite/lib/src/bindings/bindings.dart | 22 ++++++- sqlite/lib/src/bindings/signatures.dart | 7 +++ sqlite/lib/src/database.dart | 17 ++++++ sqlite/lib/src/ffi/blob.dart | 40 ++++++++++++ 8 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 moor/lib/moor_vm.dart create mode 100644 moor/lib/src/vm/vm_database.dart create mode 100644 sqlite/lib/src/ffi/blob.dart diff --git a/moor/lib/moor_vm.dart b/moor/lib/moor_vm.dart new file mode 100644 index 00000000..62e7c5ec --- /dev/null +++ b/moor/lib/moor_vm.dart @@ -0,0 +1,12 @@ +/// A version of moor that runs on the Dart VM by integrating sqlite3 with +/// ffi. +@experimental +library moor_vm; + +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:sqlite3_ffi/sqlite.dart'; +import 'moor.dart'; + +part 'src/vm/vm_database.dart'; diff --git a/moor/lib/src/vm/vm_database.dart b/moor/lib/src/vm/vm_database.dart new file mode 100644 index 00000000..20526b55 --- /dev/null +++ b/moor/lib/src/vm/vm_database.dart @@ -0,0 +1,81 @@ +part of 'package:moor/moor_vm.dart'; + +abstract class _DatabaseUser extends QueryExecutor { + final bool logStatements; + final File dbFile; + + Database _db; + + _DatabaseUser(this.logStatements, this.dbFile); + + void _logStmt(String statement, List args) { + if (logStatements) { + print('Executing $statement with variables $args'); + } + } + + @override + Future ensureOpen() { + _db ??= Database(dbFile.absolute.path); + return Future.value(true); + } + + @override + Future runCustom(String statement) { + _logStmt(statement, const []); + _db.execute(statement); + return Future.value(); + } + + Future _executeWithArgs(String statement, List args) { + _logStmt(statement, args); + _db.execute(statement, params: args); + return Future.value(_db.changes()); + } + + @override + Future runDelete(String statement, List args) { + return _executeWithArgs(statement, args); + } + + @override + Future runUpdate(String statement, List args) { + return _executeWithArgs(statement, args); + } + + @override + Future runInsert(String statement, List args) { + _logStmt(statement, args); + _db.execute(statement, params: args); + return Future.value(_db.lastInsertId()); + } + + @override + Future>> runSelect( + String statement, List args) { + if (args.isNotEmpty) { + throw UnsupportedError( + 'Select statements with variables are not yet supported.'); + } + _logStmt(statement, args); + _db.query(statement); + // todo parse rows + return Future.value([]); + } +} + +class VMDatabase extends _DatabaseUser { + VMDatabase(File file, {bool logStatements = false}) + : super(logStatements, file); + + @override + TransactionExecutor beginTransaction() { + throw UnsupportedError('Transactions are not yet supported on the Dart VM'); + } + + @override + Future runBatched(List statements) { + throw UnsupportedError( + 'Batched inserts are not yet supported on the Dart VM'); + } +} diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index 4edfcf3c..adfd7471 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -15,6 +15,9 @@ dependencies: meta: '>= 1.0.0 <2.0.0' collection: '>= 1.0.0 <2.0.0' synchronized: ^2.1.0 + # experimental ffi bindings for sqlite + sqlite3_ffi: + path: ../sqlite dev_dependencies: moor_generator: ^1.6.0 diff --git a/sqlite/README.md b/sqlite/README.md index 58542d02..e660a83b 100644 --- a/sqlite/README.md +++ b/sqlite/README.md @@ -1,3 +1,9 @@ +This was taken from [putraxor/sqlite3_ffi](https://github.com/putraxor/sqlite3_ffi), +which in turn was taken from the sqlite example in the Dart sdk repo. Moor made the following +changes: +- Support binding `Uint8List` +- Bindings for `sqlite3_changes` + # SQLite3 wrapper for dart:ffi This is an illustrative sample for how to use `dart:ffi`. diff --git a/sqlite/lib/src/bindings/bindings.dart b/sqlite/lib/src/bindings/bindings.dart index 0b15e10e..1e5c64f7 100644 --- a/sqlite/lib/src/bindings/bindings.dart +++ b/sqlite/lib/src/bindings/bindings.dart @@ -5,6 +5,7 @@ import "dart:ffi"; import "../ffi/cstring.dart"; +import '../ffi/blob.dart'; import "../ffi/dylib_utils.dart"; import "signatures.dart"; @@ -187,6 +188,13 @@ class _SQLiteBindings { /// ^The [sqlite3_reset] interface does not change the values int Function(StatementPointer statement) sqlite3_reset; + /// Returns the number of rows modified, inserted or deleted by the most + /// recently completed INSERT, UPDATE or DELETE statement on the database + /// connection [db]. + int Function(DatabasePointer db) sqlite3_changes; + + int Function(DatabasePointer db) sqlite3_last_insert_rowid; + /// Destroy A Prepared Statement Object /// /// ^The sqlite3_finalize() function is called to delete a prepared statement. @@ -332,6 +340,9 @@ class _SQLiteBindings { sqlite3_bind_int; int Function(StatementPointer statement, int columnIndex, CString value) sqlite3_bind_text; + int Function( + StatementPointer statement, int columnIndex, CBlob value, int length) + sqlite3_bind_blob; _SQLiteBindings() { sqlite = dlopenPlatformSpecific("sqlite3"); @@ -346,7 +357,9 @@ class _SQLiteBindings { sqlite3_bind_text = sqlite .lookup>("sqlite3_bind_text") .asFunction(); - + sqlite3_bind_blob = sqlite + .lookup>("sqlite3_bind_blob") + .asFunction(); sqlite3_open_v2 = sqlite .lookup>("sqlite3_open_v2") .asFunction(); @@ -363,6 +376,13 @@ class _SQLiteBindings { sqlite3_reset = sqlite .lookup>("sqlite3_reset") .asFunction(); + sqlite3_changes = sqlite + .lookup>("sqlite3_changes") + .asFunction(); + sqlite3_last_insert_rowid = sqlite + .lookup>( + "sqlite3_last_insert_rowid") + .asFunction(); sqlite3_finalize = sqlite .lookup>("sqlite3_finalize") .asFunction(); diff --git a/sqlite/lib/src/bindings/signatures.dart b/sqlite/lib/src/bindings/signatures.dart index 59b0face..1501a3dc 100644 --- a/sqlite/lib/src/bindings/signatures.dart +++ b/sqlite/lib/src/bindings/signatures.dart @@ -5,6 +5,7 @@ import "dart:ffi"; import "../ffi/cstring.dart"; +import "../ffi/blob.dart"; import "types.dart"; @@ -54,9 +55,15 @@ typedef sqlite3_column_int_native_t = Int32 Function( typedef sqlite3_column_text_native_t = CString Function( StatementPointer statement, Int32 columnIndex); +typedef sqlite3_changes_native = Int32 Function(DatabasePointer database); +typedef sqlite3_last_insert_rowid_native = Int64 Function( + DatabasePointer database); + typedef sqlite3_bind_double_native = Int32 Function( StatementPointer statement, Int32 columnIndex, Double value); typedef sqlite3_bind_int_native = Int32 Function( StatementPointer statement, Int32 columnIndex, Int32 value); typedef sqlite3_bind_text_native = Int32 Function( StatementPointer statement, Int32 columnIndex, CString value); +typedef sqlite3_bind_blob_native = Int32 Function( + StatementPointer statement, Int32 columnIndex, CBlob value, Int32 length); diff --git a/sqlite/lib/src/database.dart b/sqlite/lib/src/database.dart index c25b5cd8..61c06482 100644 --- a/sqlite/lib/src/database.dart +++ b/sqlite/lib/src/database.dart @@ -5,12 +5,14 @@ import "dart:collection"; import 'dart:convert'; import "dart:ffi"; +import 'dart:typed_data'; import "bindings/bindings.dart"; import "bindings/types.dart"; import "bindings/constants.dart"; import "collections/closable_iterator.dart"; import "ffi/cstring.dart"; +import 'ffi/blob.dart'; /// [Database] represents an open connection to a SQLite database. /// @@ -80,6 +82,11 @@ class Database { bindings.sqlite3_bind_text(statement, i + 1, text); text.free(); } + if (params[i].runtimeType.toString() == 'Uint8List') { + Uint8List param = params[i]; + final blob = CBlob.allocate(param); + bindings.sqlite3_bind_blob(statement, i + 1, blob, param.length); + } } } statementOut.free(); @@ -94,6 +101,16 @@ class Database { } } + /// Returns the number of rows modified by the most recently completed + /// INSERT, UPDATE or DELETE statement. + int changes() { + return bindings.sqlite3_changes(_database); + } + + int lastInsertId() { + return bindings.sqlite3_last_insert_rowid(_database); + } + /// Evaluate a query and return the resulting rows as an iterable. Result query(String query) { Pointer statementOut = allocate(); diff --git a/sqlite/lib/src/ffi/blob.dart b/sqlite/lib/src/ffi/blob.dart new file mode 100644 index 00000000..39472e53 --- /dev/null +++ b/sqlite/lib/src/ffi/blob.dart @@ -0,0 +1,40 @@ +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'arena.dart'; + +/// Represents a blob in C memory, managed by an [Arena]. The main difference +/// to a [CString] is that blobs aren't null-terminated. +class CBlob extends Pointer { + /// Allocates a [CBlob] in the current [Arena] and populates it with + /// [blob]. + factory CBlob(Uint8List blob) => CBlob.inArena(Arena.current(), blob); + + /// Allocates a [CString] in [arena] and populates it with [blob]. + factory CBlob.inArena(Arena arena, Uint8List blob) => + arena.scoped(CBlob.allocate(blob)); + + /// Allocate a [CBlob] not managed in and populates it with [dartBlob]. + /// + /// This [CBlob] is not managed by an [Arena]. Please ensure to [free] the + /// memory manually! + factory CBlob.allocate(Uint8List dartBlob) { + Pointer str = allocate(count: dartBlob.length); + for (int i = 0; i < dartBlob.length; ++i) { + str.elementAt(i).store(dartBlob[i]); + } + return str.cast(); + } + + /// Read the string for C memory into Dart. + static Uint8List fromC(CBlob str) { + if (str == null) return null; + int len = 0; + while (str.elementAt(++len).load() != 0); + + final Uint8List units = Uint8List(len); + for (int i = 0; i < len; ++i) units[i] = str.elementAt(i).load(); + + return units; + } +} From e00f7bfa29b0345d9e4ae121f985e68bb937c13d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 20 Jul 2019 17:20:47 +0200 Subject: [PATCH 003/117] Use our own sqlite api instead of Dart example --- moor/example/test_vm.dart | 11 + moor/lib/moor_vm.dart | 3 +- moor/lib/src/runtime/executor/executor.dart | 6 + moor/lib/src/vm/api/database.dart | 162 +++++++ moor/lib/src/vm/api/errors.dart | 27 ++ moor/lib/src/vm/api/prepared_statement.dart | 126 +++++ moor/lib/src/vm/api/result.dart | 62 +++ moor/lib/src/vm/bindings/bindings.dart | 187 ++++++++ .../lib/src/vm}/bindings/constants.dart | 4 +- .../lib/src/vm}/bindings/signatures.dart | 24 +- .../lib/src/vm}/bindings/types.dart | 4 +- moor/lib/src/vm/ffi/blob.dart | 51 +++ .../src/vm/ffi/open_platform_specific.dart | 23 + moor/lib/src/vm/vm_database.dart | 56 ++- moor/pubspec.yaml | 3 - sqlite/.gitignore | 7 - sqlite/README.md | 47 -- sqlite/analysis_options.yaml | 1 - sqlite/docs/android.md | 53 --- sqlite/docs/lib/scenario-default.svg | 130 ------ sqlite/docs/lib/scenario-full.svg | 149 ------ sqlite/docs/sqlite-tutorial.md | 234 ---------- sqlite/example/main.dart | 89 ---- sqlite/lib/sqlite.dart | 10 - sqlite/lib/src/bindings/bindings.dart | 431 ------------------ .../src/collections/closable_iterator.dart | 29 -- sqlite/lib/src/database.dart | 343 -------------- sqlite/lib/src/ffi/arena.dart | 57 --- sqlite/lib/src/ffi/blob.dart | 40 -- sqlite/lib/src/ffi/cstring.dart | 43 -- sqlite/lib/src/ffi/dylib_utils.dart | 20 - sqlite/pubspec.yaml | 9 - sqlite/test/sqlite_test.dart | 164 ------- 33 files changed, 722 insertions(+), 1883 deletions(-) create mode 100644 moor/example/test_vm.dart create mode 100644 moor/lib/src/vm/api/database.dart create mode 100644 moor/lib/src/vm/api/errors.dart create mode 100644 moor/lib/src/vm/api/prepared_statement.dart create mode 100644 moor/lib/src/vm/api/result.dart create mode 100644 moor/lib/src/vm/bindings/bindings.dart rename {sqlite/lib/src => moor/lib/src/vm}/bindings/constants.dart (97%) rename {sqlite/lib/src => moor/lib/src/vm}/bindings/signatures.dart (80%) rename {sqlite/lib/src => moor/lib/src/vm}/bindings/types.dart (98%) create mode 100644 moor/lib/src/vm/ffi/blob.dart create mode 100644 moor/lib/src/vm/ffi/open_platform_specific.dart delete mode 100644 sqlite/.gitignore delete mode 100644 sqlite/README.md delete mode 100644 sqlite/analysis_options.yaml delete mode 100644 sqlite/docs/android.md delete mode 100644 sqlite/docs/lib/scenario-default.svg delete mode 100644 sqlite/docs/lib/scenario-full.svg delete mode 100644 sqlite/docs/sqlite-tutorial.md delete mode 100644 sqlite/example/main.dart delete mode 100644 sqlite/lib/sqlite.dart delete mode 100644 sqlite/lib/src/bindings/bindings.dart delete mode 100644 sqlite/lib/src/collections/closable_iterator.dart delete mode 100644 sqlite/lib/src/database.dart delete mode 100644 sqlite/lib/src/ffi/arena.dart delete mode 100644 sqlite/lib/src/ffi/blob.dart delete mode 100644 sqlite/lib/src/ffi/cstring.dart delete mode 100644 sqlite/lib/src/ffi/dylib_utils.dart delete mode 100644 sqlite/pubspec.yaml delete mode 100644 sqlite/test/sqlite_test.dart diff --git a/moor/example/test_vm.dart b/moor/example/test_vm.dart new file mode 100644 index 00000000..543049cc --- /dev/null +++ b/moor/example/test_vm.dart @@ -0,0 +1,11 @@ +import 'dart:io'; + +import 'package:moor/moor_vm.dart'; + +void main() async { + final executor = VMDatabase(File('test.db'), logStatements: true); + + await executor.doWhenOpened((_) async { + await executor.close(); + }); +} diff --git a/moor/lib/moor_vm.dart b/moor/lib/moor_vm.dart index 62e7c5ec..1a0648e3 100644 --- a/moor/lib/moor_vm.dart +++ b/moor/lib/moor_vm.dart @@ -6,7 +6,8 @@ library moor_vm; import 'dart:io'; import 'package:meta/meta.dart'; -import 'package:sqlite3_ffi/sqlite.dart'; import 'moor.dart'; +import 'src/vm/api/database.dart'; + part 'src/vm/vm_database.dart'; diff --git a/moor/lib/src/runtime/executor/executor.dart b/moor/lib/src/runtime/executor/executor.dart index c6d83078..56097e01 100644 --- a/moor/lib/src/runtime/executor/executor.dart +++ b/moor/lib/src/runtime/executor/executor.dart @@ -44,6 +44,12 @@ abstract class QueryExecutor { /// Starts a [TransactionExecutor]. TransactionExecutor beginTransaction(); + + /// Closes this database connection. After this future completes, all further + /// calls to this executor should fail. + Future close() { + return Future.value(); + } } /// A statement that should be executed in a batch. Used internally by moor. diff --git a/moor/lib/src/vm/api/database.dart b/moor/lib/src/vm/api/database.dart new file mode 100644 index 00000000..7460c805 --- /dev/null +++ b/moor/lib/src/vm/api/database.dart @@ -0,0 +1,162 @@ +import 'dart:collection'; +import 'dart:ffi'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:collection/collection.dart'; +import 'package:moor/src/vm/bindings/constants.dart'; +import 'package:moor/src/vm/bindings/types.dart'; +import 'package:moor/src/vm/bindings/bindings.dart'; +import 'package:moor/src/vm/ffi/blob.dart'; + +part 'errors.dart'; +part 'prepared_statement.dart'; +part 'result.dart'; + +const _openingFlags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE; + +final _nullPtr = fromAddress(0); + +class Database { + final DatabasePointer _db; + final List _preparedStmt = []; + bool _isClosed; + + Database._(this._db); + + /// Opens the [file] as a sqlite3 database. The file will be created if it + /// doesn't exist. + factory Database.openFile(File file) => Database.open(file.absolute.path); + + /// Opens an in-memory sqlite3 database. + factory Database.memory() => Database.open(':memory:'); + + /// Opens an sqlite3 database from a filename. + factory Database.open(String fileName) { + final dbOut = allocate(); + final pathC = CString.allocate(fileName); + + final resultCode = + bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, _nullPtr.cast()); + final dbPointer = dbOut.load(); + + dbOut.free(); + pathC.free(); + + if (resultCode == Errors.SQLITE_OK) { + return Database._(dbPointer); + } else { + throw SqliteException._fromErrorCode(dbPointer, resultCode); + } + } + + void _ensureOpen() { + if (_isClosed) { + throw Exception('This database has already been closed'); + } + } + + /// Closes this database connection and releases the resources it uses. If + /// an error occurs while closing the database, an exception will be thrown. + /// The allocated memory will be freed either way. + void close() { + final code = bindings.sqlite3_close_v2(_db); + SqliteException exception; + if (code != Errors.SQLITE_OK) { + exception = SqliteException._fromErrorCode(_db, code); + } + _isClosed = true; + + for (var stmt in _preparedStmt) { + stmt.close(); + } + _db.free(); + + if (exception != null) { + throw exception; + } + } + + void _handleStmtFinalized(PreparedStatement stmt) { + if (!_isClosed) { + _preparedStmt.remove(stmt); + } + } + + /// Executes the [sql] statement and ignores the result. Will throw if an + /// error occurs while executing. + void execute(String sql) { + _ensureOpen(); + final sqlPtr = CString.allocate(sql); + final errorOut = allocate(); + + final result = bindings.sqlite3_exec( + _db, sqlPtr, _nullPtr.cast(), _nullPtr.cast(), errorOut); + + sqlPtr.free(); + + final errorPtr = errorOut.load(); + errorOut.free(); + + String errorMsg; + if (errorPtr.address != 0) { + errorMsg = CString.fromC(errorPtr.cast()); + // the message was allocated from sqlite, we need to free it + bindings.sqlite3_free(errorPtr.cast()); + } + + if (result != Errors.SQLITE_OK) { + throw SqliteException(errorMsg); + } + } + + /// Prepares the [sql] statement. + PreparedStatement prepare(String sql) { + _ensureOpen(); + + final stmtOut = allocate(); + final sqlPtr = CString.allocate(sql); + + final resultCode = + bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, _nullPtr.cast()); + sqlPtr.free(); + + final stmt = stmtOut.load(); + stmtOut.free(); + + if (resultCode != Errors.SQLITE_OK) { + // we don't need to worry about freeing the statement. If preparing the + // statement was unsuccessful, stmtOut.load() will be the null pointer + throw SqliteException._fromErrorCode(_db, resultCode); + } + + return PreparedStatement._(stmt, this); + } + + /// Get the application defined version of this database. + int get userVersion { + final stmt = prepare('PRAGMA user_version'); + final result = stmt.select(); + stmt.close(); + + return result.first.columnAt(0) as int; + } + + /// Update the application defined version of this database. + set userVersion(int version) { + execute('PRAGMA user_version = $version'); + } + + /// Returns the amount of rows affected by the last INSERT, UPDATE or DELETE + /// statement. + int get updatedRows { + _ensureOpen(); + return bindings.sqlite3_changes(_db); + } + + /// Returns the row-id of the last inserted row. + int get lastInsertId { + _ensureOpen(); + return bindings.sqlite3_last_insert_rowid(_db); + } +} diff --git a/moor/lib/src/vm/api/errors.dart b/moor/lib/src/vm/api/errors.dart new file mode 100644 index 00000000..59514d4c --- /dev/null +++ b/moor/lib/src/vm/api/errors.dart @@ -0,0 +1,27 @@ +part of 'database.dart'; + +class SqliteException implements Exception { + final String message; + final String explanation; + + SqliteException(this.message, [this.explanation]); + + factory SqliteException._fromErrorCode(DatabasePointer db, [int code]) { + final dbMessage = CString.fromC(bindings.sqlite3_errmsg(db).cast()); + String explanation; + if (code != null) { + explanation = CString.fromC(bindings.sqlite3_errstr(code).cast()); + } + + return SqliteException(dbMessage, explanation); + } + + @override + String toString() { + if (explanation == null) { + return 'SqliteException: $message'; + } else { + return 'SqliteException: $message, $explanation'; + } + } +} diff --git a/moor/lib/src/vm/api/prepared_statement.dart b/moor/lib/src/vm/api/prepared_statement.dart new file mode 100644 index 00000000..f09f6f35 --- /dev/null +++ b/moor/lib/src/vm/api/prepared_statement.dart @@ -0,0 +1,126 @@ +part of 'database.dart'; + +class PreparedStatement { + final StatementPointer _stmt; + final Database _db; + bool _closed = false; + + bool _bound = false; + final List _allocatedWhileBinding = []; + + PreparedStatement._(this._stmt, this._db); + + void close() { + if (!_closed) { + bindings.sqlite3_finalize(_stmt); + _db._handleStmtFinalized(this); + } + _closed = true; + } + + void _ensureNotFinalized() { + if (_closed) { + throw Exception('Tried to operate on a released prepared statement'); + } + } + + /// Executes this prepared statement as a select statement. The returned rows + /// will be returned. + Result select([List params]) { + _ensureNotFinalized(); + _bindParams(params); + + final columnCount = bindings.sqlite3_column_count(_stmt); + // not using a Map for indexed because column names are not + // guaranteed to be unique + final names = List(columnCount); + final rows = >[]; + + for (var i = 0; i < columnCount; i++) { + // name pointer doesn't need to be disposed, that happens when we finalize + names[i] = CString.fromC(bindings.sqlite3_column_name(_stmt, i).cast()); + } + + while (_step() == Errors.SQLITE_ROW) { + rows.add([for (var i = 0; i < columnCount; i++) _readValue(i)]); + } + + _reset(); + + return Result(names, rows); + } + + dynamic _readValue(int index) { + final type = bindings.sqlite3_column_type(_stmt, index); + switch (type) { + case Types.SQLITE_INTEGER: + return bindings.sqlite3_column_int(_stmt, index); + case Types.SQLITE_FLOAT: + return bindings.sqlite3_column_double(_stmt, index); + case Types.SQLITE_TEXT: + return CString.fromC(bindings.sqlite3_column_text(_stmt, index).cast()); + case Types.SQLITE_BLOB: + final length = bindings.sqlite3_column_bytes(_stmt, index); + final data = + CBlob.fromC(bindings.sqlite3_column_blob(_stmt, index), length); + return data; + case Types.SQLITE_NULL: + default: + return null; + } + } + + /// Executes this prepared statement. + void execute([List params]) { + _ensureNotFinalized(); + _bindParams(params); + + final result = _step(); + _reset(); + + if (result != Errors.SQLITE_OK || result != Errors.SQLITE_DONE) { + throw SqliteException._fromErrorCode(_db._db, result); + } + } + + void _reset() { + if (_bound) { + bindings.sqlite3_reset(_stmt); + _bound = false; + } + for (var pointer in _allocatedWhileBinding) { + pointer.free(); + } + _allocatedWhileBinding.clear(); + } + + void _bindParams(List params) { + if (params != null && params.isNotEmpty) { + // variables in sqlite are 1-indexed + for (var i = 1; i <= params.length; i++) { + final param = params[i - 1]; + + if (param == null) { + bindings.sqlite3_bind_null(_stmt, i); + } else if (param is int) { + bindings.sqlite3_bind_int(_stmt, i, param); + } else if (param is num) { + bindings.sqlite3_bind_double(_stmt, i, param.toDouble()); + } else if (param is String) { + final ptr = CString.allocate(param); + _allocatedWhileBinding.add(ptr); + + bindings.sqlite3_bind_text(_stmt, i, ptr); + } else if (param is Uint8List) { + final ptr = CBlob.allocate(param); + _allocatedWhileBinding.add(ptr); + + bindings.sqlite3_bind_blob(_stmt, i, ptr, param.length); + } + } + } + _bound = true; + } + + int _step() => bindings.sqlite3_step(_stmt); +} diff --git a/moor/lib/src/vm/api/result.dart b/moor/lib/src/vm/api/result.dart new file mode 100644 index 00000000..f7ec02a1 --- /dev/null +++ b/moor/lib/src/vm/api/result.dart @@ -0,0 +1,62 @@ +part of 'database.dart'; + +/// Stores the result of a select statement. +class Result extends Iterable { + final List columnNames; + // a result set can have multiple columns with the same name, but that's rare + // and users usually use a name as index. So we cache that for O(1) lookups + Map _calculatedIndexes; + final List> _rows; + + Result(this.columnNames, this._rows) { + _calculatedIndexes = { + for (var column in columnNames) column: columnNames.lastIndexOf(column), + }; + } + + @override + Iterator get iterator => _ResultIterator(this); +} + +/// Stores a single row in the result of a select statement. +class Row extends MapMixin + with UnmodifiableMapMixin { + final Result _result; + final int _rowIndex; + + Row._(this._result, this._rowIndex); + + /// Returns the value stored in the [i]-th column in this row (zero-indexed). + dynamic columnAt(int i) { + return _result._rows[_rowIndex][i]; + } + + @override + operator [](Object key) { + if (key is! String) return null; + + final index = _result._calculatedIndexes[key]; + if (index == null) return null; + + return columnAt(index); + } + + @override + Iterable get keys => _result.columnNames; +} + +class _ResultIterator extends Iterator { + final Result result; + int index = -1; + + _ResultIterator(this.result); + + @override + Row get current => Row._(result, index); + + @override + bool moveNext() { + index++; + return index < result._rows.length; + } +} diff --git a/moor/lib/src/vm/bindings/bindings.dart b/moor/lib/src/vm/bindings/bindings.dart new file mode 100644 index 00000000..5f6b5511 --- /dev/null +++ b/moor/lib/src/vm/bindings/bindings.dart @@ -0,0 +1,187 @@ +// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; + +import '../ffi/blob.dart'; +import '../ffi/open_platform_specific.dart'; + +import 'signatures.dart'; +import 'types.dart'; + +// ignore_for_file: comment_references, non_constant_identifier_names + +class _SQLiteBindings { + DynamicLibrary sqlite; + + int Function(CString filename, Pointer databaseOut, + int flags, CString vfs) sqlite3_open_v2; + + int Function(DatabasePointer database) sqlite3_close_v2; + void Function(Pointer ptr) sqlite3_free; + + int Function( + DatabasePointer database, + CString query, + int nbytes, + Pointer statementOut, + Pointer tail) sqlite3_prepare_v2; + + int Function( + DatabasePointer database, + CString query, + Pointer callback, + Pointer cbFirstArg, + Pointer errorMsgOut, + ) sqlite3_exec; + + int Function(StatementPointer statement) sqlite3_step; + + int Function(StatementPointer statement) sqlite3_reset; + + int Function(StatementPointer statement) sqlite3_finalize; + + int Function(StatementPointer statement) sqlite3_column_count; + + CString Function(StatementPointer statement, int columnIndex) + sqlite3_column_name; + + CString Function(StatementPointer statement, int columnIndex) + sqlite3_column_decltype; + + int Function(StatementPointer statement, int columnIndex) sqlite3_column_type; + + ValuePointer Function(StatementPointer statement, int columnIndex) + sqlite3_column_value; + double Function(StatementPointer statement, int columnIndex) + sqlite3_column_double; + int Function(StatementPointer statement, int columnIndex) sqlite3_column_int; + CString Function(StatementPointer statement, int columnIndex) + sqlite3_column_text; + CBlob Function(StatementPointer statement, int columnIndex) + sqlite3_column_blob; + + /// Returns the amount of bytes to read when using [sqlite3_column_blob]. + int Function(StatementPointer statement, int columnIndex) + sqlite3_column_bytes; + + int Function(DatabasePointer db) sqlite3_changes; + int Function(DatabasePointer db) sqlite3_last_insert_rowid; + + CString Function(int code) sqlite3_errstr; + CString Function(DatabasePointer database) sqlite3_errmsg; + + int Function(StatementPointer statement, int columnIndex, double value) + sqlite3_bind_double; + int Function(StatementPointer statement, int columnIndex, int value) + sqlite3_bind_int; + int Function(StatementPointer statement, int columnIndex, CString value) + sqlite3_bind_text; + int Function( + StatementPointer statement, int columnIndex, CBlob value, int length) + sqlite3_bind_blob; + int Function(StatementPointer statement, int columnIndex) sqlite3_bind_null; + + _SQLiteBindings() { + sqlite = dlopenPlatformSpecific('sqlite3'); + + sqlite3_bind_double = sqlite + .lookup>( + 'sqlite3_bind_double') + .asFunction(); + sqlite3_bind_int = sqlite + .lookup>('sqlite3_bind_int') + .asFunction(); + sqlite3_bind_text = sqlite + .lookup>('sqlite3_bind_text') + .asFunction(); + sqlite3_bind_blob = sqlite + .lookup>('sqlite3_bind_blob') + .asFunction(); + sqlite3_bind_null = sqlite + .lookup>('sqlite3_bind_null') + .asFunction(); + sqlite3_open_v2 = sqlite + .lookup>('sqlite3_open_v2') + .asFunction(); + sqlite3_close_v2 = sqlite + .lookup>('sqlite3_close_v2') + .asFunction(); + sqlite3_free = sqlite + .lookup>('sqlite3_free') + .asFunction(); + sqlite3_prepare_v2 = sqlite + .lookup>( + 'sqlite3_prepare_v2') + .asFunction(); + sqlite3_exec = sqlite + .lookup>('sqlite3_exec') + .asFunction(); + sqlite3_step = sqlite + .lookup>('sqlite3_step') + .asFunction(); + sqlite3_reset = sqlite + .lookup>('sqlite3_reset') + .asFunction(); + sqlite3_changes = sqlite + .lookup>('sqlite3_changes') + .asFunction(); + sqlite3_last_insert_rowid = sqlite + .lookup>( + 'sqlite3_last_insert_rowid') + .asFunction(); + sqlite3_finalize = sqlite + .lookup>('sqlite3_finalize') + .asFunction(); + sqlite3_errstr = sqlite + .lookup>('sqlite3_errstr') + .asFunction(); + sqlite3_errmsg = sqlite + .lookup>('sqlite3_errmsg') + .asFunction(); + sqlite3_column_count = sqlite + .lookup>( + 'sqlite3_column_count') + .asFunction(); + sqlite3_column_name = sqlite + .lookup>( + 'sqlite3_column_name') + .asFunction(); + sqlite3_column_decltype = sqlite + .lookup>( + 'sqlite3_column_decltype') + .asFunction(); + sqlite3_column_type = sqlite + .lookup>( + 'sqlite3_column_type') + .asFunction(); + sqlite3_column_value = sqlite + .lookup>( + 'sqlite3_column_value') + .asFunction(); + sqlite3_column_double = sqlite + .lookup>( + 'sqlite3_column_double') + .asFunction(); + sqlite3_column_int = sqlite + .lookup>( + 'sqlite3_column_int') + .asFunction(); + sqlite3_column_text = sqlite + .lookup>( + 'sqlite3_column_text') + .asFunction(); + sqlite3_column_blob = sqlite + .lookup>( + 'sqlite3_column_blob') + .asFunction(); + sqlite3_column_bytes = sqlite + .lookup>( + 'sqlite3_column_bytes') + .asFunction(); + } +} + +_SQLiteBindings _cachedBindings; +_SQLiteBindings get bindings => _cachedBindings ??= _SQLiteBindings(); diff --git a/sqlite/lib/src/bindings/constants.dart b/moor/lib/src/vm/bindings/constants.dart similarity index 97% rename from sqlite/lib/src/bindings/constants.dart rename to moor/lib/src/vm/bindings/constants.dart index 71aa82ed..d402cf11 100644 --- a/sqlite/lib/src/bindings/constants.dart +++ b/moor/lib/src/vm/bindings/constants.dart @@ -2,6 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +// ignore_for_file: constant_identifier_names + /// Result Codes /// /// Many SQLite functions return an integer result code from the set shown @@ -110,7 +112,7 @@ class Errors { /// /// These bit values are intended for use in the /// 3rd parameter to the [sqlite3_open_v2()] interface and -/// in the 4th parameter to the [sqlite3_vfs.xOpen] method. +/// in the 4th parameter to the `sqlite3_vfs.xOpen` method. class Flags { /// Ok for sqlite3_open_v2() static const int SQLITE_OPEN_READONLY = 0x00000001; diff --git a/sqlite/lib/src/bindings/signatures.dart b/moor/lib/src/vm/bindings/signatures.dart similarity index 80% rename from sqlite/lib/src/bindings/signatures.dart rename to moor/lib/src/vm/bindings/signatures.dart index 1501a3dc..e2e054ac 100644 --- a/sqlite/lib/src/bindings/signatures.dart +++ b/moor/lib/src/vm/bindings/signatures.dart @@ -2,18 +2,19 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import "dart:ffi"; +import 'dart:ffi'; -import "../ffi/cstring.dart"; -import "../ffi/blob.dart"; +import '../ffi/blob.dart'; -import "types.dart"; +import 'types.dart'; typedef sqlite3_open_v2_native_t = Int32 Function( CString filename, Pointer ppDb, Int32 flags, CString vfs); typedef sqlite3_close_v2_native_t = Int32 Function(DatabasePointer database); +typedef sqlite3_free_native = Function(Pointer pointer); + typedef sqlite3_prepare_v2_native_t = Int32 Function( DatabasePointer database, CString query, @@ -21,6 +22,13 @@ typedef sqlite3_prepare_v2_native_t = Int32 Function( Pointer statementOut, Pointer tail); +typedef sqlite3_exec_native = Int32 Function( + DatabasePointer database, + CString query, + Pointer callback, + Pointer firstCbArg, + Pointer errorOut); + typedef sqlite3_step_native_t = Int32 Function(StatementPointer statement); typedef sqlite3_reset_native_t = Int32 Function(StatementPointer statement); @@ -55,6 +63,12 @@ typedef sqlite3_column_int_native_t = Int32 Function( typedef sqlite3_column_text_native_t = CString Function( StatementPointer statement, Int32 columnIndex); +typedef sqlite3_column_blob_native_t = CBlob Function( + StatementPointer statement, Int32 columnIndex); + +typedef sqlite3_column_bytes_native_t = Int32 Function( + StatementPointer statement, Int32 columnIndex); + typedef sqlite3_changes_native = Int32 Function(DatabasePointer database); typedef sqlite3_last_insert_rowid_native = Int64 Function( DatabasePointer database); @@ -67,3 +81,5 @@ typedef sqlite3_bind_text_native = Int32 Function( StatementPointer statement, Int32 columnIndex, CString value); typedef sqlite3_bind_blob_native = Int32 Function( StatementPointer statement, Int32 columnIndex, CBlob value, Int32 length); +typedef sqlite3_bind_null_native = Int32 Function( + StatementPointer statement, Int32 columnIndex); diff --git a/sqlite/lib/src/bindings/types.dart b/moor/lib/src/vm/bindings/types.dart similarity index 98% rename from sqlite/lib/src/bindings/types.dart rename to moor/lib/src/vm/bindings/types.dart index 95035f74..b3848b27 100644 --- a/sqlite/lib/src/bindings/types.dart +++ b/moor/lib/src/vm/bindings/types.dart @@ -2,7 +2,9 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import "dart:ffi"; +import 'dart:ffi'; + +// ignore_for_file: comment_references class FunctionPointer extends Pointer {} diff --git a/moor/lib/src/vm/ffi/blob.dart b/moor/lib/src/vm/ffi/blob.dart new file mode 100644 index 00000000..cab06c5d --- /dev/null +++ b/moor/lib/src/vm/ffi/blob.dart @@ -0,0 +1,51 @@ +import 'dart:convert'; +import 'dart:ffi'; + +import 'dart:typed_data'; + +/// Pointer to arbitrary blobs that aren't null-terminated. +class CBlob extends Pointer { + /// Allocate a [CBlob] not managed in and populates it with [dartBlob]. + factory CBlob.allocate(Uint8List dartBlob) { + final ptr = allocate(count: dartBlob.length); + for (var i = 0; i < dartBlob.length; ++i) { + ptr.elementAt(i).store(dartBlob[i]); + } + return ptr.cast(); + } + + /// Read the string from C memory into Dart. + static Uint8List fromC(CBlob str, int length) { + if (str == null) return null; + assert(length >= 0); + + final units = Uint8List(length); + for (var i = 0; i < length; ++i) { + units[i] = str.elementAt(i).load(); + } + + return units; + } +} + +/// A null-terminated C string. +class CString extends Pointer { + /// Allocate a [CString] not managed in and populates it with [string]. + factory CString.allocate(String string) { + final encoded = utf8.encode(string); + final data = Uint8List(encoded.length + 1) // already filled with zeroes + ..setAll(0, encoded); + + return CBlob.allocate(data).cast(); + } + + /// Read the string from C memory into Dart. + static String fromC(CBlob str) { + if (str == null) return null; + var len = 0; + while (str.elementAt(++len).load() != 0) {} + + final list = CBlob.fromC(str, len); + return utf8.decode(list); + } +} diff --git a/moor/lib/src/vm/ffi/open_platform_specific.dart b/moor/lib/src/vm/ffi/open_platform_specific.dart new file mode 100644 index 00000000..76befe80 --- /dev/null +++ b/moor/lib/src/vm/ffi/open_platform_specific.dart @@ -0,0 +1,23 @@ +import 'dart:ffi'; +import 'dart:io'; + +String _platformPath(String name, {String path}) { + final resolvedPath = path ?? ''; + + if (Platform.isLinux || Platform.isAndroid) { + return '${resolvedPath}lib$name.so'; + } + if (Platform.isMacOS) { + return '${resolvedPath}lib$name.dylib'; + } + if (Platform.isWindows) { + return '$resolvedPath$name.dll'; + } + + throw UnsupportedError('Platform not implemented'); +} + +DynamicLibrary dlopenPlatformSpecific(String name, {String path}) { + final resolvedPath = _platformPath(name, path: path); + return DynamicLibrary.open(resolvedPath); +} diff --git a/moor/lib/src/vm/vm_database.dart b/moor/lib/src/vm/vm_database.dart index 20526b55..a9dd4e34 100644 --- a/moor/lib/src/vm/vm_database.dart +++ b/moor/lib/src/vm/vm_database.dart @@ -16,10 +16,19 @@ abstract class _DatabaseUser extends QueryExecutor { @override Future ensureOpen() { - _db ??= Database(dbFile.absolute.path); + _db ??= _openInternal(); + return Future.value(true); } + Database _openInternal() { + if (dbFile == null) { + return Database.memory(); + } else { + return Database.openFile(dbFile); + } + } + @override Future runCustom(String statement) { _logStmt(statement, const []); @@ -27,40 +36,53 @@ abstract class _DatabaseUser extends QueryExecutor { return Future.value(); } - Future _executeWithArgs(String statement, List args) { + void _runWithArgs(String statement, List args) { _logStmt(statement, args); - _db.execute(statement, params: args); - return Future.value(_db.changes()); + + if (args.isEmpty) { + _db.execute(statement); + } else { + _db.prepare(statement) + ..execute(args) + ..close(); + } + } + + Future _runAndReturnAffected(String statement, List args) { + _runWithArgs(statement, args); + return Future.value(_db.updatedRows); } @override Future runDelete(String statement, List args) { - return _executeWithArgs(statement, args); + return _runAndReturnAffected(statement, args); } @override Future runUpdate(String statement, List args) { - return _executeWithArgs(statement, args); + return _runAndReturnAffected(statement, args); } @override Future runInsert(String statement, List args) { - _logStmt(statement, args); - _db.execute(statement, params: args); - return Future.value(_db.lastInsertId()); + _runWithArgs(statement, args); + return Future.value(_db.lastInsertId); } @override Future>> runSelect( String statement, List args) { - if (args.isNotEmpty) { - throw UnsupportedError( - 'Select statements with variables are not yet supported.'); - } - _logStmt(statement, args); - _db.query(statement); - // todo parse rows - return Future.value([]); + final stmt = _db.prepare(statement); + final result = stmt.select(args); + stmt.close(); + + return Future.value(result.toList()); + } + + @override + Future close() { + _db?.close(); + return Future.value(); } } diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index adfd7471..4edfcf3c 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -15,9 +15,6 @@ dependencies: meta: '>= 1.0.0 <2.0.0' collection: '>= 1.0.0 <2.0.0' synchronized: ^2.1.0 - # experimental ffi bindings for sqlite - sqlite3_ffi: - path: ../sqlite dev_dependencies: moor_generator: ^1.6.0 diff --git a/sqlite/.gitignore b/sqlite/.gitignore deleted file mode 100644 index 7a6bc2ee..00000000 --- a/sqlite/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.dart_tool -.gdb_history -.packages -.vscode -pubspec.lock -test.db -test.db-journal \ No newline at end of file diff --git a/sqlite/README.md b/sqlite/README.md deleted file mode 100644 index e660a83b..00000000 --- a/sqlite/README.md +++ /dev/null @@ -1,47 +0,0 @@ -This was taken from [putraxor/sqlite3_ffi](https://github.com/putraxor/sqlite3_ffi), -which in turn was taken from the sqlite example in the Dart sdk repo. Moor made the following -changes: -- Support binding `Uint8List` -- Bindings for `sqlite3_changes` - -# SQLite3 wrapper for dart:ffi - -This is an illustrative sample for how to use `dart:ffi`. - - -## Building and Running this Sample - -Building and running this sample is done through pub. -Running `pub get` and `pub run example/main` should produce the following output. - -```sh -$ pub get -Resolving dependencies... (6.8s) -+ analyzer 0.35.4 -... -+ yaml 2.1.15 -Downloading analyzer 0.35.4... -Downloading kernel 0.3.14... -Downloading front_end 0.1.14... -Changed 47 dependencies! -Precompiling executables... (18.0s) -Precompiled test:test. - -``` - -``` -$ pub run example/main -1 Chocolade chip cookie Chocolade cookie foo -2 Ginger cookie null 42 -3 Cinnamon roll null null -1 Chocolade chip cookie Chocolade cookie foo -2 Ginger cookie null 42 -expected exception on accessing result data after close: The result has already been closed. -expected this query to fail: no such column: non_existing_column (Code 1: SQL logic error) -``` - -## Tutorial - -A tutorial walking through the code is available in [docs/sqlite-tutorial.md](docs/sqlite-tutorial.md). -For information on how to use this package within a Flutter app, see [docs/android.md]. -(Note: iOS is not yet supported). diff --git a/sqlite/analysis_options.yaml b/sqlite/analysis_options.yaml deleted file mode 100644 index ca56bebe..00000000 --- a/sqlite/analysis_options.yaml +++ /dev/null @@ -1 +0,0 @@ -# Using an empty analysis options so that this package gets analyzed with the less strict default rules \ No newline at end of file diff --git a/sqlite/docs/android.md b/sqlite/docs/android.md deleted file mode 100644 index 07f3e47d..00000000 --- a/sqlite/docs/android.md +++ /dev/null @@ -1,53 +0,0 @@ -**This documentation is for demonstration/testing purposes only!** - -# Using FFI with Flutter - -## Android - -Before using the FFI on Android, you need to procure an Android-compatible build of the native library you want to link against. -It's important that the shared object(s) be compatible with ABI version you wish to target (or else, that you have multiple builds for different ABIs). -See [https://developer.android.com/ndk/guides/abis] for more details on Android ABIs. -Within Flutter, the target ABI is controlled by the `--target-platform` parameter to the `flutter` command. - -The workflow for packaging a native library will depend significantly on the library itself, but to illustrate the challenges at play, we'll demonstrate how to build the SQLite library from source to use with the FFI on an Android device. - -### Building SQLite for Android - -Every Android device ships with a copy of the SQLite library (`/system/lib/libsqlite.so`). -Unfortunately, this library cannot be loaded directly by apps (see [https://developer.android.com/about/versions/nougat/android-7.0-changes#ndk]). -It is accessible only through Java. -Instead, we can build SQLite directly with the NDK. - -First, download the SQLite "amalgamation" source from [https://www.sqlite.org/download.html]. -For the sake of brevity, we'll assume the file has been saved as `sqlite-amalgamation-XXXXXXX.zip`, the Android SDK (with NDK extension) is available in `~/Android`, and we're on a Linux workstation. - -```sh -unzip sqlite-amalgamation-XXXXXXX.zip -cd sqlite-amalgamation-XXXXXXX -~/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android24-clang -c sqlite3.c -o sqlite3.o -~/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android-ld -shared sqlite3.o -o libsqlite3.so -``` - -Note the use of the `aarch64` prefix to the compiler: this indicates that we're building a shared object for the `arm64-v8a` ABI. -This will be important later. - -### Update Gradle script - -Next we need to instruct Gradle to package this library with the app, so it will be available to load off the Android device at runtime. -Create a folder `native-libraries` in the root folder of the app, and update the `android/app/build.gradle` file: - -```groovy -android { - // ... - sourceSets { - main { - jniLibs.srcDir '${project.projectDir.path}/../../native-libraries' - } - } -} -``` - -Within the `native-libraries` folder, the libraries are organized by ABI. -Therefore, we must copy the compiled `libsqlite3.so` into `native-libraries/arm64-v8a/libsqlite3.so`. -If multiple sub-directories are present, the libraries from the sub-directory corresponding to the target ABI will be available in the application's linking path, so the library can be loaded with `ffi.DynamicLibrary.open("libsqlite3.so")` in Dart. -Finally, pass `--target-platform=android-arm64` to the `flutter` command when running or building the app since `libsqlite3.so` was compiled for the `arm64-v8a` ABI. diff --git a/sqlite/docs/lib/scenario-default.svg b/sqlite/docs/lib/scenario-default.svg deleted file mode 100644 index 6ffa8a34..00000000 --- a/sqlite/docs/lib/scenario-default.svg +++ /dev/null @@ -1,130 +0,0 @@ - - - - - - - - - - - - - - - - Produced by OmniGraffle 7.9.4 - 2019-03-13 09:56:08 +0000 - - - Canvas 1 - - - Layer 1 - - - - - Flutter - App - (Imports - package) - - - - - - - Native - Library - - - - - - - dart:ffi - - - - - - - Package - API - (Does not - expose - dart:ffi) - - - - - - - Dart - - - - - - - C / C++ - - - - - - - App - Developer - - - - - - - Package - Developer - - - - - - - Dart - VM - Team - - - - - - - Bindings - - - - - - - Native - Library - Developer - - - - - - - Package - Implementation - (Code which - converts C++ - abstractions into - Dart - abstractions) - - - - - diff --git a/sqlite/docs/lib/scenario-full.svg b/sqlite/docs/lib/scenario-full.svg deleted file mode 100644 index 4ae18c50..00000000 --- a/sqlite/docs/lib/scenario-full.svg +++ /dev/null @@ -1,149 +0,0 @@ - - - - - - - - - - - - - - - - Produced by OmniGraffle 7.9.4 - 2019-03-13 09:53:08 +0000 - - - Canvas 1 - - - Layer 1 - - - - - Flutter - App - (Imports - package) - - - - - - - Native - Library - - - - - - - dart:ffi - - - - - - - Package - API - (Does not - expose - dart:ffi) - - - - - - - Dart - - - - - - - C / C++ - - - - - - - App - Developer - - - - - - - Package - Developer - - - - - - - Dart - VM - Team - - - - - - - Bindings - - - - - - - Native - Library - Developer - - - - - - - Package - Implementation - (Code which - converts C++ - abstractions into - Dart - abstractions) - - - - - - - Package - Developer - - - - - - - Glue code - (Code which - takes care of - things such as - C++ exceptions) - - - - - diff --git a/sqlite/docs/sqlite-tutorial.md b/sqlite/docs/sqlite-tutorial.md deleted file mode 100644 index c38170c9..00000000 --- a/sqlite/docs/sqlite-tutorial.md +++ /dev/null @@ -1,234 +0,0 @@ -# dart:ffi SQLite mini tutorial - -In this mini tutorial we learn how to bind SQLite, a native library, in Dart using Dart's new foreign function interface `dart:ffi`. -We build a package which provides a Dartlike SQLite API using objects and `Iterator`s. -Inside the package we write Dart code which directly invokes C functions and manipulates C memory. - -## Binding C Functions to Dart - -The first step is to load a Native Library: - -```dart -import "dart:ffi"; - -DynamicLibrary sqlite = dlopenPlatformSpecific("sqlite3"); -``` - -In a `DynamicLibrary` we can `lookup` functions. -Let's lookup the function `sqlite3_prepare_v2` in the SQLite library. -That function has the following signature in the library header file. - -```c++ -SQLITE_API int sqlite3_prepare_v2( - sqlite3 *db, /* Database handle */ - const char *zSql, /* SQL statement, UTF-8 encoded */ - int nByte, /* Maximum length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: Statement handle */ - const char **pzTail /* OUT: Pointer to unused portion of zSql */ -); -``` - -In order to lookup a function, we need a _C signature_ and a _Dart signature_. - -```dart -typedef sqlite3_prepare_v2_native_t = Int32 Function( - DatabasePointer database, - CString query, - Int32 nbytes, - Pointer statementOut, - Pointer tail); - -typedef Sqlite3_prepare_v2_t = int Function( - DatabasePointer database, - CString query, - int nbytes, - Pointer statementOut, - Pointer tail); -``` - -With these two signatures we can `lookup` the C function and expose it as a Dart function with `asFunction`. - -```dart -Sqlite3_prepare_v2_t sqlite3_prepare_v2 = sqlite - .lookup>("sqlite3_prepare_v2") - .asFunction(); -``` - -Browse the code: [platform specific dynamic library loading](../lib/src/ffi/dylib_utils.dart), [C signatures](../lib/src/bindings/signatures.dart), [Dart signatures and bindings](../lib/src/bindings/bindings.dart), and [dart:ffi dynamic library interface](../../../../sdk/lib/ffi/dynamic_library.dart). - -## Managing C Memory - -In order to call `sqlite3_prepare_v2` to prepare a SQLite statement before executing, we need to be able to pass C pointers to C functions. - -Database and Statement pointers are opaque pointers in the SQLite C API. -We specify these as classes extending `Pointer`. - -```dart -class DatabasePointer extends Pointer {} -class StatementPointer extends Pointer {} -``` - -Strings in C are pointers to character arrays. - -```dart -class CString extends Pointer {} -``` - -Pointers to C integers, floats, an doubles can be read from and written through to `dart:ffi`. -However, before we can write to C memory from dart, we need to `allocate` some memory. - -```dart -Pointer p = allocate(); // Infers type argument allocate(), and allocates 1 byte. -p.store(123); // Stores a Dart int into this C int8. -int v = p.load(); // Infers type argument p.load(), and loads a value from C memory. -``` - -Note that you can only load a Dart `int` from a C `Uint8`. -Trying to load a Dart `double` will result in a runtime exception. - -We've almost modeled C Strings. -The last thing we need is to use this `Pointer` as an array. -We can do this by using `elementAt`. - -```dart -CString string = allocate(count: 4).cast(); // Allocates 4 bytes and casts it to a string. -string.store(73); // Stores 'F' at index 0. -string.elementAt(1).store(73); // Stores 'F' at index 1. -string.elementAt(2).store(70); // Stores 'I' at index 2. -string.elementAt(3).store(0); // Null terminates the string. -``` - -We wrap the above logic of allocating strings in the constructor `CString.allocate`. - -Now we have all ingredients to call `sqlite3_prepare_v2`. - -```dart -Pointer statementOut = allocate(); -CString queryC = CString.allocate(query); -int resultCode = sqlite3_prepare_v2( - _database, queryC, -1, statementOut, fromAddress(0)); -``` - -With `dart:ffi` we are responsible for freeing C memory that we allocate. -So after calling `sqlite3_prepare_v2` we read out the statement pointer, and free the statement pointer pointer and `CString` which held the query string. - -``` -StatementPointer statement = statementOut.load(); -statementOut.free(); -queryC.free(); -``` - -Browse the code: [CString class](../lib/src/ffi/cstring.dart), [code calling sqlite3_prepare_v2](../lib/src/database.dart#57), and [dart:ffi pointer interface](../../../../sdk/lib/ffi/ffi.dart). - -## Dart API - -We would like to present the users of our package with an object oriented API - not exposing any `dart:ffi` objects to them. - -The SQLite C API returns a cursor to the first row of a result after executing a query. -We can read out the columns of this row and move the cursor to the next row. -The most natural way to expose this in Dart is through an `Iterable`. -We provide our package users with the following API. - -```dart -class Result implements Iterable {} - -class Row { - dynamic readColumnByIndex(int columnIndex) {} - dynamic readColumn(String columnName) {} -} -``` - -However, this interface does not completely match the semantics of the C API. -When we start reading the next `Row`, we do no longer have access to the previous `Row`. -We can model this by letting a `Row` keep track if its current or not. - -```dart -class Row { - bool _isCurrentRow = true; - - dynamic readColumnByIndex(int columnIndex) { - if (!_isCurrentRow) { - throw Exception( - "This row is not the current row, reading data from the non-current" - " row is not supported by sqlite."); - } - // ... - } -} -``` - -A second mismatch between Dart and C is that in C we have to manually release resources. -After executing a query and reading its results we need to call `sqlite3_finalize(statement)`. - -We can take two approaches here, either we structure the API in such a way that users of our package (implicitly) release resources, or we use finalizers to release resources. -In this tutorial we take the first approach. - -If our users iterate over all `Row`s, we can implicitly finalize the statement after they are done with the last row. -However, if they decide they do not want to iterate over the whole result, they need to explicitly state this. -In this tutorial, we use the `ClosableIterator` abstraction for `Iterators` with backing resources that need to be `close`d. - -```dart -Result result = d.query(""" - select id, name - from Cookies - ;"""); -for (Row r in result) { - String name = r.readColumn("name"); - print(name); -} -// Implicitly closes the iterator. - -result = d.query(""" - select id, name - from Cookies - ;"""); -for (Row r in result) { - int id = r.readColumn("id"); - if (id == 1) { - result.close(); // Explicitly closes the iterator, releasing underlying resources. - break; - } -} -``` - -Browse the code: [Database, Result, Row](../lib/src/database.dart), and [CloseableIterator](../lib/src/collections/closable_iterator.dart). - -## Architecture Overview - -The following diagram summarized what we have implemented as _package developers_ in this tutorial. - -![architecture](lib/scenario-default.svg) - -As the package developers wrapping an existing native library, we have only written Dart code - not any C/C++ code. -We specified bindings to the native library. -We have provided our package users with an object oriented API without exposing any `dart:ffi` objects. -And finally, we have implemented the package API by calling the C API. - -## Current dart:ffi Development Status - -In this minitutorial we used these `dart:ffi` features: - -* Loading dynamic libararies and looking up C functions in these dynamic libraries. -* Calling C functions, with `dart:ffi` automatically marshalling arguments and return value. -* Manipulating C memory through `Pointer`s with `allocate`, `free`, `load`, `store`, and `elementAt`. - -Features which we did not use in this tutorial: - -* `@struct` on subtypes of `Pointer` to define a struct with fields. (However, this feature is likely to change in the future.) - -Features which `dart:ffi` does not support yet: - -* Callbacks from C back into Dart. -* Finalizers -* C++ Exceptions (Not on roadmap yet.) - -Platform limitations: - -* `dart:ffi` is only enabled on 64 bit Windows, Linux, and MacOS. (Arm64 and 32 bit Intel are under review.) -* `dart:ffi` only works in JIT mode, not in AOT. - -It is possible to work around some of the current limitations by adding a C/C++ layer. -For example we could catch C++ exceptions in a C++ layer, and rethrow them in Dart. -The architecture diagram would change to the following in that case. - -![architecture2](lib/scenario-full.svg) \ No newline at end of file diff --git a/sqlite/example/main.dart b/sqlite/example/main.dart deleted file mode 100644 index 436a2830..00000000 --- a/sqlite/example/main.dart +++ /dev/null @@ -1,89 +0,0 @@ -import "../lib/sqlite.dart"; - -// ignore_for_file: dead_code - -void main() { - Database d = Database("test.db"); - d.execute("update Cookies set name='Changed' where name=?", - params: ['Google']); - var rows = d.query('select * from Cookies'); - rows.forEach((row) { - var id = row.readColumnByIndexAsInt(0); - var name = row.readColumnByIndexAsText(1); - print('$id $name '); - }); - return; - d.execute("drop table if exists Cookies;"); - d.execute(""" - create table Cookies ( - id integer primary key, - name text not null, - alternative_name text - );"""); - d.execute(""" - insert into Cookies (id, name, alternative_name) - values - (1,'Chocolade chip cookie', 'Chocolade cookie'), - (2,'Ginger cookie', null), - (3,'Cinnamon roll', null) - ;"""); - Result result = d.query(""" - select - id, - name, - alternative_name, - case - when id=1 then 'foo' - when id=2 then 42 - when id=3 then null - end as multi_typed_column - from Cookies - ;"""); - for (Row r in result) { - int id = r.readColumnAsInt("id"); - String name = r.readColumnByIndex(1); - String alternativeName = r.readColumn("alternative_name"); - dynamic multiTypedValue = r.readColumn("multi_typed_column"); - print("$id $name $alternativeName $multiTypedValue"); - } - result = d.query(""" - select - id, - name, - alternative_name, - case - when id=1 then 'foo' - when id=2 then 42 - when id=3 then null - end as multi_typed_column - from Cookies - ;"""); - for (Row r in result) { - int id = r.readColumnAsInt("id"); - String name = r.readColumnByIndex(1); - String alternativeName = r.readColumn("alternative_name"); - dynamic multiTypedValue = r.readColumn("multi_typed_column"); - print("$id $name $alternativeName $multiTypedValue"); - if (id == 2) { - result.close(); - break; - } - } - try { - result.iterator.moveNext(); - } on SQLiteException catch (e) { - print("expected exception on accessing result data after close: $e"); - } - try { - d.query(""" - select - id, - non_existing_column - from Cookies - ;"""); - } on SQLiteException catch (e) { - print("expected this query to fail: $e"); - } - //d.execute("drop table Cookies;"); - d.close(); -} diff --git a/sqlite/lib/sqlite.dart b/sqlite/lib/sqlite.dart deleted file mode 100644 index d6bac5be..00000000 --- a/sqlite/lib/sqlite.dart +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -/// A synchronous SQLite wrapper. -/// -/// Written using dart:ffi. -library sqlite; - -export "src/database.dart"; diff --git a/sqlite/lib/src/bindings/bindings.dart b/sqlite/lib/src/bindings/bindings.dart deleted file mode 100644 index 1e5c64f7..00000000 --- a/sqlite/lib/src/bindings/bindings.dart +++ /dev/null @@ -1,431 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import "dart:ffi"; - -import "../ffi/cstring.dart"; -import '../ffi/blob.dart'; -import "../ffi/dylib_utils.dart"; - -import "signatures.dart"; -import "types.dart"; - -class _SQLiteBindings { - DynamicLibrary sqlite; - - /// Opening A New Database Connection - /// - /// ^These routines open an SQLite database file as specified by the - /// filename argument. ^The filename argument is interpreted as UTF-8 for - /// sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte - /// order for sqlite3_open16(). ^(A database connection handle is usually - /// returned in *ppDb, even if an error occurs. The only exception is that - /// if SQLite is unable to allocate memory to hold the sqlite3 object, - /// a NULL will be written into *ppDb instead of a pointer to the sqlite3 - /// object.)^ ^(If the database is opened (and/or created) successfully, then - /// [SQLITE_OK] is returned. Otherwise an error code is returned.)^ ^The - /// [sqlite3_errmsg] or sqlite3_errmsg16() routines can be used to obtain - /// an English language description of the error following a failure of any - /// of the sqlite3_open() routines. - int Function(CString filename, Pointer databaseOut, - int flags, CString vfs) sqlite3_open_v2; - - int Function(DatabasePointer database) sqlite3_close_v2; - - /// Compiling An SQL Statement - /// - /// To execute an SQL query, it must first be compiled into a byte-code - /// program using one of these routines. - /// - /// The first argument, "db", is a database connection obtained from a - /// prior successful call to sqlite3_open, [sqlite3_open_v2] or - /// sqlite3_open16. The database connection must not have been closed. - /// - /// The second argument, "zSql", is the statement to be compiled, encoded - /// as either UTF-8 or UTF-16. The sqlite3_prepare() and sqlite3_prepare_v2() - /// interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2() - /// use UTF-16. - /// - /// ^If the nByte argument is less than zero, then zSql is read up to the - /// first zero terminator. ^If nByte is non-negative, then it is the maximum - /// number of bytes read from zSql. ^When nByte is non-negative, the - /// zSql string ends at either the first '\000' or '\u0000' character or - /// the nByte-th byte, whichever comes first. If the caller knows - /// that the supplied string is nul-terminated, then there is a small - /// performance advantage to be gained by passing an nByte parameter that - /// is equal to the number of bytes in the input string including - /// the nul-terminator bytes. - /// - /// ^If pzTail is not NULL then *pzTail is made to point to the first byte - /// past the end of the first SQL statement in zSql. These routines only - /// compile the first statement in zSql, so *pzTail is left pointing to - /// what remains uncompiled. - /// - /// ^*ppStmt is left pointing to a compiled prepared statement that can be - /// executed using sqlite3_step. ^If there is an error, *ppStmt is set - /// to NULL. ^If the input text contains no SQL (if the input is an empty - /// string or a comment) then *ppStmt is set to NULL. - /// The calling procedure is responsible for deleting the compiled - /// SQL statement using [sqlite3_finalize] after it has finished with it. - /// ppStmt may not be NULL. - /// - /// ^On success, the sqlite3_prepare family of routines return [SQLITE_OK]; - /// otherwise an error code is returned. - /// - /// The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are - /// recommended for all new programs. The two older interfaces are retained - /// for backwards compatibility, but their use is discouraged. - /// ^In the "v2" interfaces, the prepared statement - /// that is returned (the sqlite3_stmt object) contains a copy of the - /// original SQL text. This causes the [sqlite3_step] interface to - /// behave differently in three ways: - int Function( - DatabasePointer database, - CString query, - int nbytes, - Pointer statementOut, - Pointer tail) sqlite3_prepare_v2; - - /// Evaluate An SQL Statement - /// - /// After a prepared statement has been prepared using either - /// [sqlite3_prepare_v2] or sqlite3_prepare16_v2() or one of the legacy - /// interfaces sqlite3_prepare() or sqlite3_prepare16(), this function - /// must be called one or more times to evaluate the statement. - /// - /// The details of the behavior of the sqlite3_step() interface depend - /// on whether the statement was prepared using the newer "v2" interface - /// [sqlite3_prepare_v2] and sqlite3_prepare16_v2() or the older legacy - /// interface sqlite3_prepare() and sqlite3_prepare16(). The use of the - /// new "v2" interface is recommended for new applications but the legacy - /// interface will continue to be supported. - /// - /// ^In the legacy interface, the return value will be either [SQLITE_BUSY], - /// [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. - /// ^With the "v2" interface, any of the other [result codes] or - /// [extended result codes] might be returned as well. - /// - /// ^[SQLITE_BUSY] means that the database engine was unable to acquire the - /// database locks it needs to do its job. ^If the statement is a [COMMIT] - /// or occurs outside of an explicit transaction, then you can retry the - /// statement. If the statement is not a [COMMIT] and occurs within an - /// explicit transaction then you should rollback the transaction before - /// continuing. - /// - /// ^[SQLITE_DONE] means that the statement has finished executing - /// successfully. sqlite3_step() should not be called again on this virtual - /// machine without first calling [sqlite3_reset()] to reset the virtual - /// machine back to its initial state. - /// - /// ^If the SQL statement being executed returns any data, then [SQLITE_ROW] - /// is returned each time a new row of data is ready for processing by the - /// caller. The values may be accessed using the [column access functions]. - /// sqlite3_step() is called again to retrieve the next row of data. - /// - /// ^[SQLITE_ERROR] means that a run-time error (such as a constraint - /// violation) has occurred. sqlite3_step() should not be called again on - /// the VM. More information may be found by calling [sqlite3_errmsg()]. - /// ^With the legacy interface, a more specific error code (for example, - /// [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) - /// can be obtained by calling [sqlite3_reset()] on the - /// prepared statement. ^In the "v2" interface, - /// the more specific error code is returned directly by sqlite3_step(). - /// - /// [SQLITE_MISUSE] means that the this routine was called inappropriately. - /// Perhaps it was called on a prepared statement that has - /// already been [sqlite3_finalize | finalized] or on one that had - /// previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could - /// be the case that the same database connection is being used by two or - /// more threads at the same moment in time. - /// - /// For all versions of SQLite up to and including 3.6.23.1, a call to - /// [sqlite3_reset] was required after sqlite3_step() returned anything - /// other than [Errors.SQLITE_ROW] before any subsequent invocation of - /// sqlite3_step(). Failure to reset the prepared statement using - /// [sqlite3_reset()] would result in an [Errors.SQLITE_MISUSE] return from - /// sqlite3_step(). But after version 3.6.23.1, sqlite3_step() began - /// calling [sqlite3_reset] automatically in this circumstance rather - /// than returning [Errors.SQLITE_MISUSE]. This is not considered a - /// compatibility break because any application that ever receives an - /// [Errors.SQLITE_MISUSE] error is broken by definition. The - /// [SQLITE_OMIT_AUTORESET] compile-time option - /// can be used to restore the legacy behavior. - /// - /// Goofy Interface Alert: In the legacy interface, the sqlite3_step() - /// API always returns a generic error code, [SQLITE_ERROR], following any - /// error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call - /// [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the - /// specific [error codes] that better describes the error. - /// We admit that this is a goofy design. The problem has been fixed - /// with the "v2" interface. If you prepare all of your SQL statements - /// using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead - /// of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, - /// then the more specific [error codes] are returned directly - /// by sqlite3_step(). The use of the "v2" interface is recommended. - int Function(StatementPointer statement) sqlite3_step; - - /// CAPI3REF: Reset A Prepared Statement Object - /// - /// The sqlite3_reset() function is called to reset a prepared statement - /// object back to its initial state, ready to be re-executed. - /// ^Any SQL statement variables that had values bound to them using - /// the sqlite3_bind_blob | sqlite3_bind_*() API retain their values. - /// Use sqlite3_clear_bindings() to reset the bindings. - /// - /// ^The [sqlite3_reset] interface resets the prepared statement S - /// back to the beginning of its program. - /// - /// ^If the most recent call to [sqlite3_step] for the - /// prepared statement S returned [Errors.SQLITE_ROW] or [Errors.SQLITE_DONE], - /// or if [sqlite3_step] has never before been called on S, - /// then [sqlite3_reset] returns [Errors.SQLITE_OK]. - /// - /// ^If the most recent call to [sqlite3_step(S)] for the - /// prepared statement S indicated an error, then - /// [sqlite3_reset] returns an appropriate [Errors]. - /// - /// ^The [sqlite3_reset] interface does not change the values - int Function(StatementPointer statement) sqlite3_reset; - - /// Returns the number of rows modified, inserted or deleted by the most - /// recently completed INSERT, UPDATE or DELETE statement on the database - /// connection [db]. - int Function(DatabasePointer db) sqlite3_changes; - - int Function(DatabasePointer db) sqlite3_last_insert_rowid; - - /// Destroy A Prepared Statement Object - /// - /// ^The sqlite3_finalize() function is called to delete a prepared statement. - /// ^If the most recent evaluation of the statement encountered no errors - /// or if the statement is never been evaluated, then sqlite3_finalize() - /// returns SQLITE_OK. ^If the most recent evaluation of statement S failed, - /// then sqlite3_finalize(S) returns the appropriate error code or extended - /// error code. - /// - /// ^The sqlite3_finalize(S) routine can be called at any point during - /// the life cycle of prepared statement S: - /// before statement S is ever evaluated, after - /// one or more calls to [sqlite3_reset], or after any call - /// to [sqlite3_step] regardless of whether or not the statement has - /// completed execution. - /// - /// ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. - /// - /// The application must finalize every prepared statement in order to avoid - /// resource leaks. It is a grievous error for the application to try to use - /// a prepared statement after it has been finalized. Any use of a prepared - /// statement after it has been finalized can result in undefined and - /// undesirable behavior such as segfaults and heap corruption. - int Function(StatementPointer statement) sqlite3_finalize; - - /// Number Of Columns In A Result Set - /// - /// ^Return the number of columns in the result set returned by the - /// prepared statement. ^This routine returns 0 if pStmt is an SQL - /// statement that does not return data (for example an [UPDATE]). - int Function(StatementPointer statement) sqlite3_column_count; - - /// Column Names In A Result Set - /// - /// ^These routines return the name assigned to a particular column - /// in the result set of a SELECT statement. ^The sqlite3_column_name() - /// interface returns a pointer to a zero-terminated UTF-8 string - /// and sqlite3_column_name16() returns a pointer to a zero-terminated - /// UTF-16 string. ^The first parameter is the prepared statement - /// that implements the SELECT statement. ^The second parameter is the - /// column number. ^The leftmost column is number 0. - /// - /// ^The returned string pointer is valid until either the prepared statement - /// is destroyed by [sqlite3_finalize] or until the statement is automatically - /// reprepared by the first call to [sqlite3_step] for a particular run - /// or until the next call to - /// sqlite3_column_name() or sqlite3_column_name16() on the same column. - /// - /// ^If sqlite3_malloc() fails during the processing of either routine - /// (for example during a conversion from UTF-8 to UTF-16) then a - /// NULL pointer is returned. - /// - /// ^The name of a result column is the value of the "AS" clause for - /// that column, if there is an AS clause. If there is no AS clause - /// then the name of the column is unspecified and may change from - CString Function(StatementPointer statement, int columnIndex) - sqlite3_column_name; - - /// CAPI3REF: Declared Datatype Of A Query Result - /// - /// ^(The first parameter is a prepared statement. - /// If this statement is a SELECT statement and the Nth column of the - /// returned result set of that SELECT is a table column (not an - /// expression or subquery) then the declared type of the table - /// column is returned.)^ ^If the Nth column of the result set is an - /// expression or subquery, then a NULL pointer is returned. - /// ^The returned string is always UTF-8 encoded. - /// - /// ^(For example, given the database schema: - /// - /// CREATE TABLE t1(c1 VARIANT); - /// - /// and the following statement to be compiled: - /// - /// SELECT c1 + 1, c1 FROM t1; - /// - /// this routine would return the string "VARIANT" for the second result - /// column (i==1), and a NULL pointer for the first result column (i==0).)^ - /// - /// ^SQLite uses dynamic run-time typing. ^So just because a column - /// is declared to contain a particular type does not mean that the - /// data stored in that column is of the declared type. SQLite is - /// strongly typed, but the typing is dynamic not static. ^Type - /// is associated with individual values, not with the containers - /// used to hold those values. - CString Function(StatementPointer statement, int columnIndex) - sqlite3_column_decltype; - - int Function(StatementPointer statement, int columnIndex) sqlite3_column_type; - - ValuePointer Function(StatementPointer statement, int columnIndex) - sqlite3_column_value; - - double Function(StatementPointer statement, int columnIndex) - sqlite3_column_double; - - int Function(StatementPointer statement, int columnIndex) sqlite3_column_int; - - CString Function(StatementPointer statement, int columnIndex) - sqlite3_column_text; - - /// The sqlite3_errstr() interface returns the English-language text that - /// describes the result code, as UTF-8. Memory to hold the error message - /// string is managed internally and must not be freed by the application. - CString Function(int code) sqlite3_errstr; - - /// Error Codes And Messages - /// - /// ^The sqlite3_errcode() interface returns the numeric [result code] or - /// [extended result code] for the most recent failed sqlite3_* API call - /// associated with a [database connection]. If a prior API call failed - /// but the most recent API call succeeded, the return value from - /// sqlite3_errcode() is undefined. ^The sqlite3_extended_errcode() - /// interface is the same except that it always returns the - /// [extended result code] even when extended result codes are - /// disabled. - /// - /// ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language - /// text that describes the error, as either UTF-8 or UTF-16 respectively. - /// ^(Memory to hold the error message string is managed internally. - /// The application does not need to worry about freeing the result. - /// However, the error string might be overwritten or deallocated by - /// subsequent calls to other SQLite interface functions.)^ - /// - /// When the serialized [threading mode] is in use, it might be the - /// case that a second error occurs on a separate thread in between - /// the time of the first error and the call to these interfaces. - /// When that happens, the second error will be reported since these - /// interfaces always report the most recent result. To avoid - /// this, each thread can obtain exclusive use of the [database connection] D - /// by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning - /// to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after - /// all calls to the interfaces listed here are completed. - /// - /// If an interface fails with SQLITE_MISUSE, that means the interface - /// was invoked incorrectly by the application. In that case, the - /// error code and message may or may not be set. - CString Function(DatabasePointer database) sqlite3_errmsg; - - int Function(StatementPointer statement, int columnIndex, double value) - sqlite3_bind_double; - int Function(StatementPointer statement, int columnIndex, int value) - sqlite3_bind_int; - int Function(StatementPointer statement, int columnIndex, CString value) - sqlite3_bind_text; - int Function( - StatementPointer statement, int columnIndex, CBlob value, int length) - sqlite3_bind_blob; - - _SQLiteBindings() { - sqlite = dlopenPlatformSpecific("sqlite3"); - - sqlite3_bind_double = sqlite - .lookup>( - "sqlite3_bind_double") - .asFunction(); - sqlite3_bind_int = sqlite - .lookup>("sqlite3_bind_int") - .asFunction(); - sqlite3_bind_text = sqlite - .lookup>("sqlite3_bind_text") - .asFunction(); - sqlite3_bind_blob = sqlite - .lookup>("sqlite3_bind_blob") - .asFunction(); - sqlite3_open_v2 = sqlite - .lookup>("sqlite3_open_v2") - .asFunction(); - sqlite3_close_v2 = sqlite - .lookup>("sqlite3_close_v2") - .asFunction(); - sqlite3_prepare_v2 = sqlite - .lookup>( - "sqlite3_prepare_v2") - .asFunction(); - sqlite3_step = sqlite - .lookup>("sqlite3_step") - .asFunction(); - sqlite3_reset = sqlite - .lookup>("sqlite3_reset") - .asFunction(); - sqlite3_changes = sqlite - .lookup>("sqlite3_changes") - .asFunction(); - sqlite3_last_insert_rowid = sqlite - .lookup>( - "sqlite3_last_insert_rowid") - .asFunction(); - sqlite3_finalize = sqlite - .lookup>("sqlite3_finalize") - .asFunction(); - sqlite3_errstr = sqlite - .lookup>("sqlite3_errstr") - .asFunction(); - sqlite3_errmsg = sqlite - .lookup>("sqlite3_errmsg") - .asFunction(); - sqlite3_column_count = sqlite - .lookup>( - "sqlite3_column_count") - .asFunction(); - sqlite3_column_name = sqlite - .lookup>( - "sqlite3_column_name") - .asFunction(); - sqlite3_column_decltype = sqlite - .lookup>( - "sqlite3_column_decltype") - .asFunction(); - sqlite3_column_type = sqlite - .lookup>( - "sqlite3_column_type") - .asFunction(); - sqlite3_column_value = sqlite - .lookup>( - "sqlite3_column_value") - .asFunction(); - sqlite3_column_double = sqlite - .lookup>( - "sqlite3_column_double") - .asFunction(); - sqlite3_column_int = sqlite - .lookup>( - "sqlite3_column_int") - .asFunction(); - sqlite3_column_text = sqlite - .lookup>( - "sqlite3_column_text") - .asFunction(); - } -} - -_SQLiteBindings _cachedBindings; -_SQLiteBindings get bindings => _cachedBindings ??= _SQLiteBindings(); diff --git a/sqlite/lib/src/collections/closable_iterator.dart b/sqlite/lib/src/collections/closable_iterator.dart deleted file mode 100644 index a86a58b2..00000000 --- a/sqlite/lib/src/collections/closable_iterator.dart +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -/// This iterator should be [close]d after use. -/// -/// [ClosableIterator]s often use resources which should be freed after use. -/// The consumer of the iterator can either manually [close] the iterator, or -/// consume all elements on which the iterator will automatically be closed. -abstract class ClosableIterator extends Iterator { - /// Close this iterator. - void close(); - - /// Moves to the next element and [close]s the iterator if it was the last - /// element. - bool moveNext(); -} - -/// This iterable's iterator should be [close]d after use. -/// -/// Companion class of [ClosableIterator]. -abstract class ClosableIterable extends Iterable { - /// Close this iterables iterator. - void close(); - - /// Returns a [ClosableIterator] that allows iterating the elements of this - /// [ClosableIterable]. - ClosableIterator get iterator; -} diff --git a/sqlite/lib/src/database.dart b/sqlite/lib/src/database.dart deleted file mode 100644 index 61c06482..00000000 --- a/sqlite/lib/src/database.dart +++ /dev/null @@ -1,343 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import "dart:collection"; -import 'dart:convert'; -import "dart:ffi"; -import 'dart:typed_data'; - -import "bindings/bindings.dart"; -import "bindings/types.dart"; -import "bindings/constants.dart"; -import "collections/closable_iterator.dart"; -import "ffi/cstring.dart"; -import 'ffi/blob.dart'; - -/// [Database] represents an open connection to a SQLite database. -/// -/// All functions against a database may throw [SQLiteError]. -/// -/// This database interacts with SQLite synchonously. -class Database { - DatabasePointer _database; - bool _open = false; - - /// Open a database located at the file [path]. - Database(String path, - [int flags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE]) { - Pointer dbOut = allocate(); - CString pathC = CString.allocate(path); - final int resultCode = - bindings.sqlite3_open_v2(pathC, dbOut, flags, fromAddress(0)); - _database = dbOut.load(); - dbOut.free(); - pathC.free(); - - if (resultCode == Errors.SQLITE_OK) { - _open = true; - } else { - // Even if "open" fails, sqlite3 will still create a database object. We - // can just destroy it. - SQLiteException exception = _loadError(resultCode); - close(); - throw exception; - } - } - - /// Close the database. - /// - /// This should only be called once on a database unless an exception is - /// thrown. It should be called at least once to finalize the database and - /// avoid resource leaks. - void close() { - assert(_open); - final int resultCode = bindings.sqlite3_close_v2(_database); - if (resultCode == Errors.SQLITE_OK) { - _open = false; - } else { - throw _loadError(resultCode); - } - } - - /// Execute a query, discarding any returned rows. - void execute(String query, {List params}) { - Pointer statementOut = allocate(); - CString queryC = CString.allocate(query); - int resultCode = bindings.sqlite3_prepare_v2( - _database, queryC, -1, statementOut, fromAddress(0)); - - StatementPointer statement = statementOut.load(); - if (params != null) { - for (var i = 0; i < params.length; i++) { - if (params[i].runtimeType.toString() == 'int') { - bindings.sqlite3_bind_int(statement, i + 1, params[i]); - } - if (params[i].runtimeType.toString() == 'double') { - bindings.sqlite3_bind_double(statement, i + 1, params[i]); - } - if (params[i].runtimeType.toString() == 'String') { - String param = utf8.decode(utf8.encode(params[i])); - var text = CString.allocate(param); - bindings.sqlite3_bind_text(statement, i + 1, text); - text.free(); - } - if (params[i].runtimeType.toString() == 'Uint8List') { - Uint8List param = params[i]; - final blob = CBlob.allocate(param); - bindings.sqlite3_bind_blob(statement, i + 1, blob, param.length); - } - } - } - statementOut.free(); - queryC.free(); - - while (resultCode == Errors.SQLITE_ROW || resultCode == Errors.SQLITE_OK) { - resultCode = bindings.sqlite3_step(statement); - } - bindings.sqlite3_finalize(statement); - if (resultCode != Errors.SQLITE_DONE) { - throw _loadError(resultCode); - } - } - - /// Returns the number of rows modified by the most recently completed - /// INSERT, UPDATE or DELETE statement. - int changes() { - return bindings.sqlite3_changes(_database); - } - - int lastInsertId() { - return bindings.sqlite3_last_insert_rowid(_database); - } - - /// Evaluate a query and return the resulting rows as an iterable. - Result query(String query) { - Pointer statementOut = allocate(); - CString queryC = CString.allocate(query); - int resultCode = bindings.sqlite3_prepare_v2( - _database, queryC, -1, statementOut, fromAddress(0)); - StatementPointer statement = statementOut.load(); - - statementOut.free(); - queryC.free(); - - if (resultCode != Errors.SQLITE_OK) { - bindings.sqlite3_finalize(statement); - throw _loadError(resultCode); - } - - Map columnIndices = {}; - int columnCount = bindings.sqlite3_column_count(statement); - for (int i = 0; i < columnCount; i++) { - String columnName = - CString.fromUtf8(bindings.sqlite3_column_name(statement, i)); - columnIndices[columnName] = i; - } - - return Result._(this, statement, columnIndices); - } - - SQLiteException _loadError([int errorCode]) { - String errorMessage = CString.fromUtf8(bindings.sqlite3_errmsg(_database)); - if (errorCode == null) { - return SQLiteException(errorMessage); - } - String errorCodeExplanation = - CString.fromUtf8(bindings.sqlite3_errstr(errorCode)); - return SQLiteException( - "$errorMessage (Code $errorCode: $errorCodeExplanation)"); - } -} - -/// [Result] represents a [Database.query]'s result and provides an [Iterable] -/// interface for the results to be consumed. -/// -/// Please note that this iterator should be [close]d manually if not all [Row]s -/// are consumed. -class Result extends IterableBase implements ClosableIterable { - final Database _database; - final ClosableIterator _iterator; - final StatementPointer _statement; - final Map _columnIndices; - - Row _currentRow = null; - - Result._( - this._database, - this._statement, - this._columnIndices, - ) : _iterator = _ResultIterator(_statement, _columnIndices) {} - - void close() => _iterator.close(); - - ClosableIterator get iterator => _iterator; -} - -class _ResultIterator implements ClosableIterator { - final StatementPointer _statement; - final Map _columnIndices; - - Row _currentRow = null; - bool _closed = false; - - _ResultIterator(this._statement, this._columnIndices) {} - - bool moveNext() { - if (_closed) { - throw SQLiteException("The result has already been closed."); - } - _currentRow?._setNotCurrent(); - int stepResult = bindings.sqlite3_step(_statement); - if (stepResult == Errors.SQLITE_ROW) { - _currentRow = Row._(_statement, _columnIndices); - return true; - } else { - close(); - return false; - } - } - - Row get current { - if (_closed) { - throw SQLiteException("The result has already been closed."); - } - return _currentRow; - } - - void close() { - _currentRow?._setNotCurrent(); - _closed = true; - bindings.sqlite3_finalize(_statement); - } -} - -class Row { - final StatementPointer _statement; - final Map _columnIndices; - - bool _isCurrentRow = true; - - Row._(this._statement, this._columnIndices) {} - - /// Reads column [columnName]. - /// - /// By default it returns a dynamically typed value. If [convert] is set to - /// [Convert.StaticType] the value is converted to the static type computed - /// for the column by the query compiler. - dynamic readColumn(String columnName, - {Convert convert = Convert.DynamicType}) { - return readColumnByIndex(_columnIndices[columnName], convert: convert); - } - - /// Reads column [columnName]. - /// - /// By default it returns a dynamically typed value. If [convert] is set to - /// [Convert.StaticType] the value is converted to the static type computed - /// for the column by the query compiler. - dynamic readColumnByIndex(int columnIndex, - {Convert convert = Convert.DynamicType}) { - _checkIsCurrentRow(); - - Type dynamicType; - if (convert == Convert.DynamicType) { - dynamicType = - _typeFromCode(bindings.sqlite3_column_type(_statement, columnIndex)); - } else { - dynamicType = _typeFromText(CString.fromUtf8( - bindings.sqlite3_column_decltype(_statement, columnIndex))); - } - - switch (dynamicType) { - case Type.Integer: - return readColumnByIndexAsInt(columnIndex); - case Type.Text: - return readColumnByIndexAsText(columnIndex); - case Type.Null: - return null; - break; - default: - } - } - - /// Reads column [columnName] and converts to [Type.Integer] if not an - /// integer. - int readColumnAsInt(String columnName) { - return readColumnByIndexAsInt(_columnIndices[columnName]); - } - - /// Reads column [columnIndex] and converts to [Type.Integer] if not an - /// integer. - int readColumnByIndexAsInt(int columnIndex) { - _checkIsCurrentRow(); - return bindings.sqlite3_column_int(_statement, columnIndex); - } - - /// Reads column [columnName] and converts to [Type.Text] if not text. - String readColumnAsText(String columnName) { - return readColumnByIndexAsText(_columnIndices[columnName]); - } - - /// Reads column [columnIndex] and converts to [Type.Text] if not text. - String readColumnByIndexAsText(int columnIndex) { - _checkIsCurrentRow(); - return CString.fromUtf8( - bindings.sqlite3_column_text(_statement, columnIndex)); - } - - void _checkIsCurrentRow() { - if (!_isCurrentRow) { - throw Exception( - "This row is not the current row, reading data from the non-current" - " row is not supported by sqlite."); - } - } - - void _setNotCurrent() { - _isCurrentRow = false; - } -} - -Type _typeFromCode(int code) { - switch (code) { - case Types.SQLITE_INTEGER: - return Type.Integer; - case Types.SQLITE_FLOAT: - return Type.Float; - case Types.SQLITE_TEXT: - return Type.Text; - case Types.SQLITE_BLOB: - return Type.Blob; - case Types.SQLITE_NULL: - return Type.Null; - } - throw Exception("Unknown type [$code]"); -} - -Type _typeFromText(String textRepresentation) { - switch (textRepresentation) { - case "integer": - return Type.Integer; - case "float": - return Type.Float; - case "text": - return Type.Text; - case "blob": - return Type.Blob; - case "null": - return Type.Null; - } - if (textRepresentation == null) return Type.Null; - throw Exception("Unknown type [$textRepresentation]"); -} - -enum Type { Integer, Float, Text, Blob, Null } - -enum Convert { DynamicType, StaticType } - -class SQLiteException { - final String message; - SQLiteException(this.message); - - String toString() => message; -} diff --git a/sqlite/lib/src/ffi/arena.dart b/sqlite/lib/src/ffi/arena.dart deleted file mode 100644 index 2e19d55f..00000000 --- a/sqlite/lib/src/ffi/arena.dart +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import "dart:async"; -import "dart:ffi"; - -/// [Arena] manages allocated C memory. -/// -/// Arenas are zoned. -class Arena { - Arena(); - - List> _allocations = []; - - /// Bound the lifetime of [ptr] to this [Arena]. - T scoped(T ptr) { - _allocations.add(ptr.cast()); - return ptr; - } - - /// Frees all memory pointed to by [Pointer]s in this arena. - void finalize() { - for (final ptr in _allocations) { - ptr.free(); - } - } - - /// The last [Arena] in the zone. - factory Arena.current() { - return Zone.current[#_currentArena]; - } -} - -/// Bound the lifetime of [ptr] to the current [Arena]. -T scoped(T ptr) => Arena.current().scoped(ptr); - -class RethrownError { - dynamic original; - StackTrace originalStackTrace; - RethrownError(this.original, this.originalStackTrace); - toString() => """RethrownError(${original}) -${originalStackTrace}"""; -} - -/// Runs the [body] in an [Arena] freeing all memory which is [scoped] during -/// execution of [body] at the end of the execution. -R runArena(R Function(Arena) body) { - Arena arena = Arena(); - try { - return runZoned(() => body(arena), - zoneValues: {#_currentArena: arena}, - onError: (error, st) => throw RethrownError(error, st)); - } finally { - arena.finalize(); - } -} diff --git a/sqlite/lib/src/ffi/blob.dart b/sqlite/lib/src/ffi/blob.dart deleted file mode 100644 index 39472e53..00000000 --- a/sqlite/lib/src/ffi/blob.dart +++ /dev/null @@ -1,40 +0,0 @@ -import 'dart:ffi'; -import 'dart:typed_data'; - -import 'arena.dart'; - -/// Represents a blob in C memory, managed by an [Arena]. The main difference -/// to a [CString] is that blobs aren't null-terminated. -class CBlob extends Pointer { - /// Allocates a [CBlob] in the current [Arena] and populates it with - /// [blob]. - factory CBlob(Uint8List blob) => CBlob.inArena(Arena.current(), blob); - - /// Allocates a [CString] in [arena] and populates it with [blob]. - factory CBlob.inArena(Arena arena, Uint8List blob) => - arena.scoped(CBlob.allocate(blob)); - - /// Allocate a [CBlob] not managed in and populates it with [dartBlob]. - /// - /// This [CBlob] is not managed by an [Arena]. Please ensure to [free] the - /// memory manually! - factory CBlob.allocate(Uint8List dartBlob) { - Pointer str = allocate(count: dartBlob.length); - for (int i = 0; i < dartBlob.length; ++i) { - str.elementAt(i).store(dartBlob[i]); - } - return str.cast(); - } - - /// Read the string for C memory into Dart. - static Uint8List fromC(CBlob str) { - if (str == null) return null; - int len = 0; - while (str.elementAt(++len).load() != 0); - - final Uint8List units = Uint8List(len); - for (int i = 0; i < len; ++i) units[i] = str.elementAt(i).load(); - - return units; - } -} diff --git a/sqlite/lib/src/ffi/cstring.dart b/sqlite/lib/src/ffi/cstring.dart deleted file mode 100644 index da095e47..00000000 --- a/sqlite/lib/src/ffi/cstring.dart +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import "dart:convert"; -import "dart:ffi"; - -import "arena.dart"; - -/// Represents a String in C memory, managed by an [Arena]. -class CString extends Pointer { - /// Allocates a [CString] in the current [Arena] and populates it with - /// [dartStr]. - factory CString(String dartStr) => CString.inArena(Arena.current(), dartStr); - - /// Allocates a [CString] in [arena] and populates it with [dartStr]. - factory CString.inArena(Arena arena, String dartStr) => - arena.scoped(CString.allocate(dartStr)); - - /// Allocate a [CString] not managed in and populates it with [dartStr]. - /// - /// This [CString] is not managed by an [Arena]. Please ensure to [free] the - /// memory manually! - factory CString.allocate(String dartStr) { - List units = Utf8Encoder().convert(dartStr); - Pointer str = allocate(count: units.length + 1); - for (int i = 0; i < units.length; ++i) { - str.elementAt(i).store(units[i]); - } - str.elementAt(units.length).store(0); - return str.cast(); - } - - /// Read the string for C memory into Dart. - static String fromUtf8(CString str) { - if (str == null) return null; - int len = 0; - while (str.elementAt(++len).load() != 0); - List units = List(len); - for (int i = 0; i < len; ++i) units[i] = str.elementAt(i).load(); - return Utf8Decoder().convert(units); - } -} diff --git a/sqlite/lib/src/ffi/dylib_utils.dart b/sqlite/lib/src/ffi/dylib_utils.dart deleted file mode 100644 index fb8153af..00000000 --- a/sqlite/lib/src/ffi/dylib_utils.dart +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:ffi' as ffi; -import 'dart:io' show Platform; - -String _platformPath(String name, {String path}) { - if (path == null) path = ""; - if (Platform.isLinux || Platform.isAndroid) - return path + "lib" + name + ".so"; - if (Platform.isMacOS) return path + "lib" + name + ".dylib"; - if (Platform.isWindows) return path + name + ".dll"; - throw Exception("Platform not implemented"); -} - -ffi.DynamicLibrary dlopenPlatformSpecific(String name, {String path}) { - String fullPath = _platformPath(name, path: path); - return ffi.DynamicLibrary.open(fullPath); -} diff --git a/sqlite/pubspec.yaml b/sqlite/pubspec.yaml deleted file mode 100644 index efcf9d41..00000000 --- a/sqlite/pubspec.yaml +++ /dev/null @@ -1,9 +0,0 @@ -name: sqlite3_ffi -version: 0.0.1 -description: >- - Sqlite3 wrapper for dart:ffi. -author: Daco Harkes , Samir Jindel -environment: - sdk: '>=2.1.0 <3.0.0' -dev_dependencies: - test: ^1.5.3 \ No newline at end of file diff --git a/sqlite/test/sqlite_test.dart b/sqlite/test/sqlite_test.dart deleted file mode 100644 index 95bbc6c1..00000000 --- a/sqlite/test/sqlite_test.dart +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -// VMOptions=--optimization-counter-threshold=5 - -import "package:test/test.dart"; - -import '../lib/sqlite.dart'; - -void main() { - test("sqlite integration test", () { - Database d = Database("test.db"); - d.execute("drop table if exists Cookies;"); - d.execute(""" - create table Cookies ( - id integer primary key, - name text not null, - alternative_name text - );"""); - d.execute(""" - insert into Cookies (id, name, alternative_name) - values - (1,'Chocolade chip cookie', 'Chocolade cookie'), - (2,'Ginger cookie', null), - (3,'Cinnamon roll', null) - ;"""); - Result result = d.query(""" - select - id, - name, - alternative_name, - case - when id=1 then 'foo' - when id=2 then 42 - when id=3 then null - end as multi_typed_column - from Cookies - ;"""); - for (Row r in result) { - int id = r.readColumnAsInt("id"); - expect(true, 1 <= id && id <= 3); - String name = r.readColumnByIndex(1); - expect(true, name is String); - String alternativeName = r.readColumn("alternative_name"); - expect(true, alternativeName is String || alternativeName == null); - dynamic multiTypedValue = r.readColumn("multi_typed_column"); - expect( - true, - multiTypedValue == 42 || - multiTypedValue == 'foo' || - multiTypedValue == null); - print("$id $name $alternativeName $multiTypedValue"); - } - result = d.query(""" - select - id, - name, - alternative_name, - case - when id=1 then 'foo' - when id=2 then 42 - when id=3 then null - end as multi_typed_column - from Cookies - ;"""); - for (Row r in result) { - int id = r.readColumnAsInt("id"); - expect(true, 1 <= id && id <= 3); - String name = r.readColumnByIndex(1); - expect(true, name is String); - String alternativeName = r.readColumn("alternative_name"); - expect(true, alternativeName is String || alternativeName == null); - dynamic multiTypedValue = r.readColumn("multi_typed_column"); - expect( - true, - multiTypedValue == 42 || - multiTypedValue == 'foo' || - multiTypedValue == null); - print("$id $name $alternativeName $multiTypedValue"); - if (id == 2) { - result.close(); - break; - } - } - try { - result.iterator.moveNext(); - } on SQLiteException catch (e) { - print("expected exception on accessing result data after close: $e"); - } - try { - d.query(""" - select - id, - non_existing_column - from Cookies - ;"""); - } on SQLiteException catch (e) { - print("expected this query to fail: $e"); - } - d.execute("drop table Cookies;"); - d.close(); - }); - - test("concurrent db open and queries", () { - Database d = Database("test.db"); - Database d2 = Database("test.db"); - d.execute("drop table if exists Cookies;"); - d.execute(""" - create table Cookies ( - id integer primary key, - name text not null, - alternative_name text - );"""); - d.execute(""" - insert into Cookies (id, name, alternative_name) - values - (1,'Chocolade chip cookie', 'Chocolade cookie'), - (2,'Ginger cookie', null), - (3,'Cinnamon roll', null) - ;"""); - Result r = d.query("select * from Cookies;"); - Result r2 = d2.query("select * from Cookies;"); - r.iterator..moveNext(); - r2.iterator..moveNext(); - r.iterator..moveNext(); - Result r3 = d2.query("select * from Cookies;"); - r3.iterator..moveNext(); - expect(2, r.iterator.current.readColumn("id")); - expect(1, r2.iterator.current.readColumn("id")); - expect(1, r3.iterator.current.readColumn("id")); - r.close(); - r2.close(); - r3.close(); - d.close(); - d2.close(); - }); - - test("stress test", () { - Database d = Database("test.db"); - d.execute("drop table if exists Cookies;"); - d.execute(""" - create table Cookies ( - id integer primary key, - name text not null, - alternative_name text - );"""); - int repeats = 100; - for (int i = 0; i < repeats; i++) { - d.execute(""" - insert into Cookies (name, alternative_name) - values - ('Chocolade chip cookie', 'Chocolade cookie'), - ('Ginger cookie', null), - ('Cinnamon roll', null) - ;"""); - } - Result r = d.query("select count(*) from Cookies;"); - int count = r.first.readColumnByIndexAsInt(0); - expect(count, 3 * repeats); - r.close(); - d.close(); - }); -} From e9cba8d6000f4eed3a03a1af4df308a1356c0fd1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 20 Jul 2019 19:23:38 +0200 Subject: [PATCH 004/117] Type fixes for ffi --- moor/example/test_vm.dart | 11 ----------- moor/lib/src/vm/api/database.dart | 16 +++++++--------- moor/lib/src/vm/bindings/bindings.dart | 6 +++--- moor/lib/src/vm/bindings/signatures.dart | 6 +++--- moor/lib/src/vm/ffi/blob.dart | 2 +- moor/lib/src/vm/vm_database.dart | 2 ++ 6 files changed, 16 insertions(+), 27 deletions(-) delete mode 100644 moor/example/test_vm.dart diff --git a/moor/example/test_vm.dart b/moor/example/test_vm.dart deleted file mode 100644 index 543049cc..00000000 --- a/moor/example/test_vm.dart +++ /dev/null @@ -1,11 +0,0 @@ -import 'dart:io'; - -import 'package:moor/moor_vm.dart'; - -void main() async { - final executor = VMDatabase(File('test.db'), logStatements: true); - - await executor.doWhenOpened((_) async { - await executor.close(); - }); -} diff --git a/moor/lib/src/vm/api/database.dart b/moor/lib/src/vm/api/database.dart index 7460c805..b64aba38 100644 --- a/moor/lib/src/vm/api/database.dart +++ b/moor/lib/src/vm/api/database.dart @@ -15,12 +15,10 @@ part 'result.dart'; const _openingFlags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE; -final _nullPtr = fromAddress(0); - class Database { final DatabasePointer _db; final List _preparedStmt = []; - bool _isClosed; + bool _isClosed = false; Database._(this._db); @@ -37,7 +35,7 @@ class Database { final pathC = CString.allocate(fileName); final resultCode = - bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, _nullPtr.cast()); + bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, fromAddress(0)); final dbPointer = dbOut.load(); dbOut.free(); @@ -91,15 +89,15 @@ class Database { final errorOut = allocate(); final result = bindings.sqlite3_exec( - _db, sqlPtr, _nullPtr.cast(), _nullPtr.cast(), errorOut); + _db, sqlPtr, fromAddress(0), fromAddress(0), errorOut); sqlPtr.free(); - final errorPtr = errorOut.load(); + final errorPtr = errorOut.load(); errorOut.free(); String errorMsg; - if (errorPtr.address != 0) { + if (errorPtr != null) { errorMsg = CString.fromC(errorPtr.cast()); // the message was allocated from sqlite, we need to free it bindings.sqlite3_free(errorPtr.cast()); @@ -118,7 +116,7 @@ class Database { final sqlPtr = CString.allocate(sql); final resultCode = - bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, _nullPtr.cast()); + bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, fromAddress(0)); sqlPtr.free(); final stmt = stmtOut.load(); @@ -126,7 +124,7 @@ class Database { if (resultCode != Errors.SQLITE_OK) { // we don't need to worry about freeing the statement. If preparing the - // statement was unsuccessful, stmtOut.load() will be the null pointer + // statement was unsuccessful, stmtOut.load() will be null throw SqliteException._fromErrorCode(_db, resultCode); } diff --git a/moor/lib/src/vm/bindings/bindings.dart b/moor/lib/src/vm/bindings/bindings.dart index 5f6b5511..fa41a1fb 100644 --- a/moor/lib/src/vm/bindings/bindings.dart +++ b/moor/lib/src/vm/bindings/bindings.dart @@ -31,9 +31,9 @@ class _SQLiteBindings { int Function( DatabasePointer database, CString query, - Pointer callback, - Pointer cbFirstArg, - Pointer errorMsgOut, + Pointer callback, + Pointer cbFirstArg, + Pointer errorMsgOut, ) sqlite3_exec; int Function(StatementPointer statement) sqlite3_step; diff --git a/moor/lib/src/vm/bindings/signatures.dart b/moor/lib/src/vm/bindings/signatures.dart index e2e054ac..34d0cc6c 100644 --- a/moor/lib/src/vm/bindings/signatures.dart +++ b/moor/lib/src/vm/bindings/signatures.dart @@ -13,7 +13,7 @@ typedef sqlite3_open_v2_native_t = Int32 Function( typedef sqlite3_close_v2_native_t = Int32 Function(DatabasePointer database); -typedef sqlite3_free_native = Function(Pointer pointer); +typedef sqlite3_free_native = Void Function(Pointer pointer); typedef sqlite3_prepare_v2_native_t = Int32 Function( DatabasePointer database, @@ -25,8 +25,8 @@ typedef sqlite3_prepare_v2_native_t = Int32 Function( typedef sqlite3_exec_native = Int32 Function( DatabasePointer database, CString query, - Pointer callback, - Pointer firstCbArg, + Pointer callback, + Pointer firstCbArg, Pointer errorOut); typedef sqlite3_step_native_t = Int32 Function(StatementPointer statement); diff --git a/moor/lib/src/vm/ffi/blob.dart b/moor/lib/src/vm/ffi/blob.dart index cab06c5d..c85dc6b3 100644 --- a/moor/lib/src/vm/ffi/blob.dart +++ b/moor/lib/src/vm/ffi/blob.dart @@ -7,7 +7,7 @@ import 'dart:typed_data'; class CBlob extends Pointer { /// Allocate a [CBlob] not managed in and populates it with [dartBlob]. factory CBlob.allocate(Uint8List dartBlob) { - final ptr = allocate(count: dartBlob.length); + final ptr = allocate(count: dartBlob.length); for (var i = 0; i < dartBlob.length; ++i) { ptr.elementAt(i).store(dartBlob[i]); } diff --git a/moor/lib/src/vm/vm_database.dart b/moor/lib/src/vm/vm_database.dart index a9dd4e34..9be23d60 100644 --- a/moor/lib/src/vm/vm_database.dart +++ b/moor/lib/src/vm/vm_database.dart @@ -90,6 +90,8 @@ class VMDatabase extends _DatabaseUser { VMDatabase(File file, {bool logStatements = false}) : super(logStatements, file); + VMDatabase.memory({bool logStatements = false}) : super(logStatements, null); + @override TransactionExecutor beginTransaction() { throw UnsupportedError('Transactions are not yet supported on the Dart VM'); From 722900238314dc403535e9e5e44392b8e54b4cdb Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 20 Jul 2019 20:47:48 +0200 Subject: [PATCH 005/117] Integration test for the Dart VM --- moor/lib/src/vm/api/prepared_statement.dart | 15 ++-- moor/lib/src/vm/bindings/bindings.dart | 9 ++- moor/lib/src/vm/bindings/signatures.dart | 8 +-- moor/lib/src/vm/ffi/blob.dart | 4 ++ moor/lib/src/vm/vm_database.dart | 60 +++++++++++----- moor/test/vm/integration_test.dart | 79 +++++++++++++++++++++ 6 files changed, 143 insertions(+), 32 deletions(-) create mode 100644 moor/test/vm/integration_test.dart diff --git a/moor/lib/src/vm/api/prepared_statement.dart b/moor/lib/src/vm/api/prepared_statement.dart index f09f6f35..b528a781 100644 --- a/moor/lib/src/vm/api/prepared_statement.dart +++ b/moor/lib/src/vm/api/prepared_statement.dart @@ -12,6 +12,7 @@ class PreparedStatement { void close() { if (!_closed) { + _reset(); bindings.sqlite3_finalize(_stmt); _db._handleStmtFinalized(this); } @@ -28,6 +29,7 @@ class PreparedStatement { /// will be returned. Result select([List params]) { _ensureNotFinalized(); + _reset(); _bindParams(params); final columnCount = bindings.sqlite3_column_count(_stmt); @@ -45,8 +47,6 @@ class PreparedStatement { rows.add([for (var i = 0; i < columnCount; i++) _readValue(i)]); } - _reset(); - return Result(names, rows); } @@ -73,12 +73,12 @@ class PreparedStatement { /// Executes this prepared statement. void execute([List params]) { _ensureNotFinalized(); + _reset(); _bindParams(params); final result = _step(); - _reset(); - if (result != Errors.SQLITE_OK || result != Errors.SQLITE_DONE) { + if (result != Errors.SQLITE_OK && result != Errors.SQLITE_DONE) { throw SqliteException._fromErrorCode(_db._db, result); } } @@ -110,12 +110,15 @@ class PreparedStatement { final ptr = CString.allocate(param); _allocatedWhileBinding.add(ptr); - bindings.sqlite3_bind_text(_stmt, i, ptr); + bindings.sqlite3_bind_text(_stmt, i, ptr, -1, fromAddress(0)); } else if (param is Uint8List) { + // todo we just have a null pointer param.isEmpty. I guess we have + // to use sqlite3_bind_zeroblob for that? final ptr = CBlob.allocate(param); _allocatedWhileBinding.add(ptr); - bindings.sqlite3_bind_blob(_stmt, i, ptr, param.length); + bindings.sqlite3_bind_blob( + _stmt, i, ptr, param.length, fromAddress(0)); } } } diff --git a/moor/lib/src/vm/bindings/bindings.dart b/moor/lib/src/vm/bindings/bindings.dart index fa41a1fb..9d13e022 100644 --- a/moor/lib/src/vm/bindings/bindings.dart +++ b/moor/lib/src/vm/bindings/bindings.dart @@ -76,11 +76,10 @@ class _SQLiteBindings { sqlite3_bind_double; int Function(StatementPointer statement, int columnIndex, int value) sqlite3_bind_int; - int Function(StatementPointer statement, int columnIndex, CString value) - sqlite3_bind_text; - int Function( - StatementPointer statement, int columnIndex, CBlob value, int length) - sqlite3_bind_blob; + int Function(StatementPointer statement, int columnIndex, CString value, + int minusOne, Pointer disposeCb) sqlite3_bind_text; + int Function(StatementPointer statement, int columnIndex, CBlob value, + int length, Pointer disposeCb) sqlite3_bind_blob; int Function(StatementPointer statement, int columnIndex) sqlite3_bind_null; _SQLiteBindings() { diff --git a/moor/lib/src/vm/bindings/signatures.dart b/moor/lib/src/vm/bindings/signatures.dart index 34d0cc6c..8e834f30 100644 --- a/moor/lib/src/vm/bindings/signatures.dart +++ b/moor/lib/src/vm/bindings/signatures.dart @@ -77,9 +77,9 @@ typedef sqlite3_bind_double_native = Int32 Function( StatementPointer statement, Int32 columnIndex, Double value); typedef sqlite3_bind_int_native = Int32 Function( StatementPointer statement, Int32 columnIndex, Int32 value); -typedef sqlite3_bind_text_native = Int32 Function( - StatementPointer statement, Int32 columnIndex, CString value); -typedef sqlite3_bind_blob_native = Int32 Function( - StatementPointer statement, Int32 columnIndex, CBlob value, Int32 length); +typedef sqlite3_bind_text_native = Int32 Function(StatementPointer statement, + Int32 columnIndex, CString value, Int32 length, Pointer callback); +typedef sqlite3_bind_blob_native = Int32 Function(StatementPointer statement, + Int32 columnIndex, CBlob value, Int32 length, Pointer callback); typedef sqlite3_bind_null_native = Int32 Function( StatementPointer statement, Int32 columnIndex); diff --git a/moor/lib/src/vm/ffi/blob.dart b/moor/lib/src/vm/ffi/blob.dart index c85dc6b3..b29ea671 100644 --- a/moor/lib/src/vm/ffi/blob.dart +++ b/moor/lib/src/vm/ffi/blob.dart @@ -7,6 +7,10 @@ import 'dart:typed_data'; class CBlob extends Pointer { /// Allocate a [CBlob] not managed in and populates it with [dartBlob]. factory CBlob.allocate(Uint8List dartBlob) { + if (dartBlob.isEmpty) { + return fromAddress(0); + } + final ptr = allocate(count: dartBlob.length); for (var i = 0; i < dartBlob.length; ++i) { ptr.elementAt(i).store(dartBlob[i]); diff --git a/moor/lib/src/vm/vm_database.dart b/moor/lib/src/vm/vm_database.dart index 9be23d60..1cad00f0 100644 --- a/moor/lib/src/vm/vm_database.dart +++ b/moor/lib/src/vm/vm_database.dart @@ -14,21 +14,6 @@ abstract class _DatabaseUser extends QueryExecutor { } } - @override - Future ensureOpen() { - _db ??= _openInternal(); - - return Future.value(true); - } - - Database _openInternal() { - if (dbFile == null) { - return Database.memory(); - } else { - return Database.openFile(dbFile); - } - } - @override Future runCustom(String statement) { _logStmt(statement, const []); @@ -72,6 +57,7 @@ abstract class _DatabaseUser extends QueryExecutor { @override Future>> runSelect( String statement, List args) { + _logStmt(statement, args); final stmt = _db.prepare(statement); final result = stmt.select(args); stmt.close(); @@ -92,6 +78,37 @@ class VMDatabase extends _DatabaseUser { VMDatabase.memory({bool logStatements = false}) : super(logStatements, null); + @override + Future ensureOpen() async { + if (_db == null) { + _db = _openInternal(); + await _runMigrations(); + } + return true; + } + + Database _openInternal() { + if (dbFile == null) { + return Database.memory(); + } else { + return Database.openFile(dbFile); + } + } + + Future _runMigrations() async { + final current = _db.userVersion; + final target = databaseInfo.schemaVersion; + + if (current == 0) { + await databaseInfo.handleDatabaseCreation(executor: runCustom); + } else if (current < target) { + await databaseInfo.handleDatabaseVersionChange( + executor: null, from: current, to: target); + } + + _db.userVersion = target; + } + @override TransactionExecutor beginTransaction() { throw UnsupportedError('Transactions are not yet supported on the Dart VM'); @@ -99,7 +116,16 @@ class VMDatabase extends _DatabaseUser { @override Future runBatched(List statements) { - throw UnsupportedError( - 'Batched inserts are not yet supported on the Dart VM'); + for (var stmt in statements) { + final prepared = _db.prepare(stmt.sql); + + for (var boundVars in stmt.variables) { + prepared.execute(boundVars); + } + + prepared.close(); + } + + return Future.value(); } } diff --git a/moor/test/vm/integration_test.dart b/moor/test/vm/integration_test.dart new file mode 100644 index 00000000..8adca5e1 --- /dev/null +++ b/moor/test/vm/integration_test.dart @@ -0,0 +1,79 @@ +import 'package:moor/moor.dart'; +import 'package:test_api/test_api.dart'; +import 'package:moor/moor_vm.dart'; + +import '../data/tables/todos.dart'; + +TodoDb db; + +void main() { + test('CRUD integration test', () async { + db = TodoDb(VMDatabase.memory(logStatements: false)); + + // write some dummy data + await insertCategory(); + await insertUser(); + await insertTodos(); + + await db.into(db.sharedTodos).insert(SharedTodo(todo: 2, user: 1)); + + // test select statements + final forUser = (await db.someDao.todosForUser(1)).single; + expect(forUser.title, 'Another entry'); + + // test delete statements + await db.deleteTodoById(2); + final queryAgain = await db.someDao.todosForUser(1); + expect(queryAgain, isEmpty); + + // test update statements + await (db.update(db.todosTable)..where((t) => t.id.equals(1))) + .write(const TodosTableCompanion(content: Value('Updated content'))); + final readUpdated = await db.select(db.todosTable).getSingle(); + expect(readUpdated.content, 'Updated content'); + }); +} + +Future insertCategory() async { + final forInsert = const CategoriesCompanion(description: Value('Work')); + final row = Category(id: 1, description: 'Work'); + + final id = await db.into(db.categories).insert(forInsert); + expect(id, equals(1)); + + final loaded = await db.select(db.categories).getSingle(); + expect(loaded, equals(row)); +} + +Future insertUser() async { + final profilePic = Uint8List.fromList([1, 2, 3, 4, 5, 6]); + final forInsert = UsersCompanion( + name: const Value('Dashy McDashface'), + isAwesome: const Value(true), + profilePicture: Value(profilePic), + ); + + final id = await db.into(db.users).insert(forInsert); + expect(id, equals(1)); + + final user = await db.select(db.users).getSingle(); + expect(user.id, equals(1)); + expect(user.name, equals('Dashy McDashface')); + expect(user.isAwesome, isTrue); + expect(user.profilePicture, profilePic); +} + +Future insertTodos() async { + await db.into(db.todosTable).insertAll([ + TodosTableCompanion( + title: const Value('A first entry'), + content: const Value('Some content I guess'), + targetDate: Value(DateTime(2019)), + ), + const TodosTableCompanion( + title: Value('Another entry'), + content: Value('this is a really creative test case'), + category: Value(1), // "Work" + ), + ]); +} From 4b9cd084b5c8466ac17d438b05b69f567b7ebd3f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 20 Jul 2019 22:55:27 +0200 Subject: [PATCH 006/117] VM: transactions and before open callback --- moor/lib/moor_vm.dart | 2 + moor/lib/src/runtime/executor/executor.dart | 22 +- moor/lib/src/vm/vm_database.dart | 210 ++++++++++++++------ moor/lib/src/web/web_db.dart | 19 +- moor_flutter/lib/moor_flutter.dart | 18 +- 5 files changed, 171 insertions(+), 100 deletions(-) diff --git a/moor/lib/moor_vm.dart b/moor/lib/moor_vm.dart index 1a0648e3..8a5abfcc 100644 --- a/moor/lib/moor_vm.dart +++ b/moor/lib/moor_vm.dart @@ -3,9 +3,11 @@ @experimental library moor_vm; +import 'dart:async'; import 'dart:io'; import 'package:meta/meta.dart'; +import 'package:synchronized/synchronized.dart'; import 'moor.dart'; import 'src/vm/api/database.dart'; diff --git a/moor/lib/src/runtime/executor/executor.dart b/moor/lib/src/runtime/executor/executor.dart index 56097e01..584ba5b0 100644 --- a/moor/lib/src/runtime/executor/executor.dart +++ b/moor/lib/src/runtime/executor/executor.dart @@ -85,8 +85,28 @@ class BatchedStatement { } /// A [QueryExecutor] that runs multiple queries atomically. -abstract class TransactionExecutor extends QueryExecutor { +mixin TransactionExecutor on QueryExecutor { /// Completes the transaction. No further queries may be sent to to this /// [QueryExecutor] after this method was called. Future send(); + + @override + TransactionExecutor beginTransaction() { + throw UnsupportedError( + 'Transactions cannot be created inside a transaction'); + } +} + +/// Used internally by moor. Responsible for executing the `beforeOpen` +/// callback. +mixin BeforeOpenMixin on QueryExecutor { + @override + Future ensureOpen() { + return Future.value(true); + } + + @override + TransactionExecutor beginTransaction() { + throw UnsupportedError('Transactions cannot be created inside beforeOpen!'); + } } diff --git a/moor/lib/src/vm/vm_database.dart b/moor/lib/src/vm/vm_database.dart index 1cad00f0..fde16c31 100644 --- a/moor/lib/src/vm/vm_database.dart +++ b/moor/lib/src/vm/vm_database.dart @@ -1,24 +1,44 @@ part of 'package:moor/moor_vm.dart'; -abstract class _DatabaseUser extends QueryExecutor { +class _DbState { final bool logStatements; - final File dbFile; + final File file; + final Lock lock = Lock(); - Database _db; + Database db; - _DatabaseUser(this.logStatements, this.dbFile); + _DbState(this.logStatements, this.file); +} + +abstract class _DatabaseUser extends QueryExecutor { + final _DbState _state; + + bool get _bypassLock => false; + Database get _db => _state.db; + + _DatabaseUser(this._state); void _logStmt(String statement, List args) { - if (logStatements) { + if (_state.logStatements) { print('Executing $statement with variables $args'); } } + Future _synchronized(FutureOr computation()) async { + final lock = _state.lock; + if (_bypassLock) { + return await computation(); + } + + return await lock.synchronized(computation); + } + @override Future runCustom(String statement) { - _logStmt(statement, const []); - _db.execute(statement); - return Future.value(); + return _synchronized(() { + _logStmt(statement, const []); + _db.execute(statement); + }); } void _runWithArgs(String statement, List args) { @@ -40,78 +60,37 @@ abstract class _DatabaseUser extends QueryExecutor { @override Future runDelete(String statement, List args) { - return _runAndReturnAffected(statement, args); + return _synchronized(() { + return _runAndReturnAffected(statement, args); + }); } @override Future runUpdate(String statement, List args) { - return _runAndReturnAffected(statement, args); + return _synchronized(() { + return _runAndReturnAffected(statement, args); + }); } @override Future runInsert(String statement, List args) { - _runWithArgs(statement, args); - return Future.value(_db.lastInsertId); + return _synchronized(() { + _runWithArgs(statement, args); + return Future.value(_db.lastInsertId); + }); } @override Future>> runSelect( String statement, List args) { - _logStmt(statement, args); - final stmt = _db.prepare(statement); - final result = stmt.select(args); - stmt.close(); + return _synchronized(() { + _logStmt(statement, args); + final stmt = _db.prepare(statement); + final result = stmt.select(args); + stmt.close(); - return Future.value(result.toList()); - } - - @override - Future close() { - _db?.close(); - return Future.value(); - } -} - -class VMDatabase extends _DatabaseUser { - VMDatabase(File file, {bool logStatements = false}) - : super(logStatements, file); - - VMDatabase.memory({bool logStatements = false}) : super(logStatements, null); - - @override - Future ensureOpen() async { - if (_db == null) { - _db = _openInternal(); - await _runMigrations(); - } - return true; - } - - Database _openInternal() { - if (dbFile == null) { - return Database.memory(); - } else { - return Database.openFile(dbFile); - } - } - - Future _runMigrations() async { - final current = _db.userVersion; - final target = databaseInfo.schemaVersion; - - if (current == 0) { - await databaseInfo.handleDatabaseCreation(executor: runCustom); - } else if (current < target) { - await databaseInfo.handleDatabaseVersionChange( - executor: null, from: current, to: target); - } - - _db.userVersion = target; - } - - @override - TransactionExecutor beginTransaction() { - throw UnsupportedError('Transactions are not yet supported on the Dart VM'); + return Future.value(result.toList()); + }); } @override @@ -129,3 +108,102 @@ class VMDatabase extends _DatabaseUser { return Future.value(); } } + +class VMDatabase extends _DatabaseUser { + VMDatabase(File file, {bool logStatements = false}) + : super(_DbState(logStatements, file)); + + VMDatabase.memory({bool logStatements = false}) + : this(null, logStatements: logStatements); + + @override + Future ensureOpen() async { + if (_db == null) { + _state.db = _openInternal(); + await _runMigrations(); + } + return true; + } + + Database _openInternal() { + if (_state.file == null) { + return Database.memory(); + } else { + return Database.openFile(_state.file); + } + } + + Future _runMigrations() async { + final current = _db.userVersion; + final target = databaseInfo.schemaVersion; + + if (current == 0) { + await databaseInfo.handleDatabaseCreation(executor: runCustom); + } else if (current < target) { + await databaseInfo.handleDatabaseVersionChange( + executor: null, from: current, to: target); + } + + _db.userVersion = target; + + await _synchronized(() { + databaseInfo.beforeOpenCallback( + _BeforeOpenExecutor(_state), OpeningDetails(current, target)); + }); + } + + @override + Future close() { + _db?.close(); + return Future.value(); + } + + @override + TransactionExecutor beginTransaction() { + final transactionReady = Completer(); + final executor = _TransactionExecutor(_state, transactionReady.future); + + _synchronized(() async { + // we have the lock, so start the transaction + transactionReady.complete(true); + await executor.completed; + }); + + return executor; + } +} + +class _BeforeOpenExecutor extends _DatabaseUser with BeforeOpenMixin { + @override + final bool _bypassLock = true; + _BeforeOpenExecutor(_DbState state) : super(state); +} + +class _TransactionExecutor extends _DatabaseUser with TransactionExecutor { + @override + final bool _bypassLock = true; + final Future _openingFuture; + bool _sentBeginTransaction = false; + + final Completer _completer = Completer(); + Future get completed => _completer.future; + + _TransactionExecutor(_DbState state, this._openingFuture) : super(state); + + @override + Future ensureOpen() async { + await _openingFuture; + if (!_sentBeginTransaction) { + _db.execute('BEGIN TRANSACTION'); + _sentBeginTransaction = true; + } + return Future.value(true); + } + + @override + Future send() { + _db.execute('COMMIT TRANSACTION;'); + _completer.complete(); + return Future.value(); + } +} diff --git a/moor/lib/src/web/web_db.dart b/moor/lib/src/web/web_db.dart index fd72ba53..6e3dc966 100644 --- a/moor/lib/src/web/web_db.dart +++ b/moor/lib/src/web/web_db.dart @@ -228,24 +228,14 @@ class WebDatabase extends _DatabaseUser { } } -class _BeforeOpenExecutor extends _DatabaseUser { +class _BeforeOpenExecutor extends _DatabaseUser with BeforeOpenMixin { _BeforeOpenExecutor(_DbState state) : super(state); @override final bool _bypassLock = true; - - @override - TransactionExecutor beginTransaction() { - throw UnsupportedError( - "Transactions aren't supported in the before open callback"); - } - - @override - Future ensureOpen() => Future.value(true); } -class _TransactionExecutor extends _DatabaseUser - implements TransactionExecutor { +class _TransactionExecutor extends _DatabaseUser with TransactionExecutor { _TransactionExecutor(_DbState state, this._openingFuture) : super(state); @override @@ -264,11 +254,6 @@ class _TransactionExecutor extends _DatabaseUser _needsSave = true; } - @override - TransactionExecutor beginTransaction() { - throw UnsupportedError('Cannot have nested transactions'); - } - @override Future ensureOpen() async { await _openingFuture; diff --git a/moor_flutter/lib/moor_flutter.dart b/moor_flutter/lib/moor_flutter.dart index 8e9d528d..26ff54e6 100644 --- a/moor_flutter/lib/moor_flutter.dart +++ b/moor_flutter/lib/moor_flutter.dart @@ -160,7 +160,7 @@ class FlutterQueryExecutor extends _DatabaseOwner { } class _SqfliteTransactionExecutor extends _DatabaseOwner - implements TransactionExecutor { + with TransactionExecutor { @override s.Transaction db; @@ -194,11 +194,6 @@ class _SqfliteTransactionExecutor extends _DatabaseOwner openingCompleter.future, actionCompleter, sendFuture, db.logStatements); } - @override - TransactionExecutor beginTransaction() { - throw StateError('Transactions cannot create another transaction!'); - } - @override Future ensureOpen() => _open.then((_) => true); @@ -209,18 +204,9 @@ class _SqfliteTransactionExecutor extends _DatabaseOwner } } -class _BeforeOpenExecutor extends _DatabaseOwner { +class _BeforeOpenExecutor extends _DatabaseOwner with BeforeOpenMixin { @override final s.DatabaseExecutor db; _BeforeOpenExecutor(this.db, bool logStatements) : super(logStatements); - - @override - TransactionExecutor beginTransaction() { - throw UnsupportedError( - "Transactions can't be started in the befoeOpen callback"); - } - - @override - Future ensureOpen() => Future.value(true); } From 56f8e447bd269bc9824563aac22c5a312256096e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 22 Jul 2019 12:11:38 +0200 Subject: [PATCH 007/117] Migrate VM database to use the new delegate api --- moor/lib/moor_vm.dart | 2 +- moor/lib/src/runtime/database.dart | 6 + moor/lib/src/runtime/executor/executor.dart | 22 +- .../src/runtime/executor/helpers/engines.dart | 6 + moor/lib/src/vm/api/result.dart | 8 +- moor/lib/src/vm/vm_database.dart | 230 +++++------------- moor/lib/src/web/web_db.dart | 18 +- moor/test/vm/integration_test.dart | 26 ++ 8 files changed, 127 insertions(+), 191 deletions(-) diff --git a/moor/lib/moor_vm.dart b/moor/lib/moor_vm.dart index 8a5abfcc..314ec4f5 100644 --- a/moor/lib/moor_vm.dart +++ b/moor/lib/moor_vm.dart @@ -7,7 +7,7 @@ import 'dart:async'; import 'dart:io'; import 'package:meta/meta.dart'; -import 'package:synchronized/synchronized.dart'; +import 'backends.dart'; import 'moor.dart'; import 'src/vm/api/database.dart'; diff --git a/moor/lib/src/runtime/database.dart b/moor/lib/src/runtime/database.dart index 41045a69..912d6ffc 100644 --- a/moor/lib/src/runtime/database.dart +++ b/moor/lib/src/runtime/database.dart @@ -321,4 +321,10 @@ abstract class GeneratedDatabase extends DatabaseConnectionUser }); } } + + /// Closes this database instance and released the resources associated with + /// it. + void close() { + executor.close(); + } } diff --git a/moor/lib/src/runtime/executor/executor.dart b/moor/lib/src/runtime/executor/executor.dart index d5d60a20..3869d341 100644 --- a/moor/lib/src/runtime/executor/executor.dart +++ b/moor/lib/src/runtime/executor/executor.dart @@ -89,28 +89,8 @@ class BatchedStatement { } /// A [QueryExecutor] that runs multiple queries atomically. -mixin TransactionExecutor on QueryExecutor { +abstract class TransactionExecutor extends QueryExecutor { /// Completes the transaction. No further queries may be sent to to this /// [QueryExecutor] after this method was called. Future send(); - - @override - TransactionExecutor beginTransaction() { - throw UnsupportedError( - 'Transactions cannot be created inside a transaction'); - } -} - -/// Used internally by moor. Responsible for executing the `beforeOpen` -/// callback. -mixin BeforeOpenMixin on QueryExecutor { - @override - Future ensureOpen() { - return Future.value(true); - } - - @override - TransactionExecutor beginTransaction() { - throw UnsupportedError('Transactions cannot be created inside beforeOpen!'); - } } diff --git a/moor/lib/src/runtime/executor/helpers/engines.dart b/moor/lib/src/runtime/executor/helpers/engines.dart index 0561be05..4777f031 100644 --- a/moor/lib/src/runtime/executor/helpers/engines.dart +++ b/moor/lib/src/runtime/executor/helpers/engines.dart @@ -217,6 +217,12 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { final alreadyOpen = await delegate.isOpen; if (alreadyOpen) return true; + // could have changed already, see https://github.com/dart-lang/linter/issues/1384 + // ignore: invariant_booleans + if (_openingCompleter != null) { + return _openingCompleter.future; + } + // not already open or opening. Open the database now! _openingCompleter = Completer(); await delegate.open(databaseInfo); diff --git a/moor/lib/src/vm/api/result.dart b/moor/lib/src/vm/api/result.dart index f7ec02a1..3204124a 100644 --- a/moor/lib/src/vm/api/result.dart +++ b/moor/lib/src/vm/api/result.dart @@ -6,9 +6,9 @@ class Result extends Iterable { // a result set can have multiple columns with the same name, but that's rare // and users usually use a name as index. So we cache that for O(1) lookups Map _calculatedIndexes; - final List> _rows; + final List> rows; - Result(this.columnNames, this._rows) { + Result(this.columnNames, this.rows) { _calculatedIndexes = { for (var column in columnNames) column: columnNames.lastIndexOf(column), }; @@ -28,7 +28,7 @@ class Row extends MapMixin /// Returns the value stored in the [i]-th column in this row (zero-indexed). dynamic columnAt(int i) { - return _result._rows[_rowIndex][i]; + return _result.rows[_rowIndex][i]; } @override @@ -57,6 +57,6 @@ class _ResultIterator extends Iterator { @override bool moveNext() { index++; - return index < result._rows.length; + return index < result.rows.length; } } diff --git a/moor/lib/src/vm/vm_database.dart b/moor/lib/src/vm/vm_database.dart index fde16c31..23af5542 100644 --- a/moor/lib/src/vm/vm_database.dart +++ b/moor/lib/src/vm/vm_database.dart @@ -1,96 +1,47 @@ part of 'package:moor/moor_vm.dart'; -class _DbState { - final bool logStatements; - final File file; - final Lock lock = Lock(); +/// A moor database that runs on the Dart VM. +class VMDatabase extends DelegatedDatabase { + VMDatabase._(DatabaseDelegate delegate, bool logStatements) + : super(delegate, isSequential: true, logStatements: logStatements); - Database db; + /// Creates a database that will store its result in the [file], creating it + /// if it doesn't exist. + factory VMDatabase(File file, {bool logStatements = false}) { + return VMDatabase._(_VmDelegate(file), logStatements); + } - _DbState(this.logStatements, this.file); + /// Creates a database won't persist its changes on disk. + factory VMDatabase.memory({bool logStatements = false}) { + return VMDatabase._(_VmDelegate(null), logStatements); + } } -abstract class _DatabaseUser extends QueryExecutor { - final _DbState _state; +class _VmDelegate extends DatabaseDelegate { + Database _db; - bool get _bypassLock => false; - Database get _db => _state.db; + final File file; - _DatabaseUser(this._state); - - void _logStmt(String statement, List args) { - if (_state.logStatements) { - print('Executing $statement with variables $args'); - } - } - - Future _synchronized(FutureOr computation()) async { - final lock = _state.lock; - if (_bypassLock) { - return await computation(); - } - - return await lock.synchronized(computation); - } + _VmDelegate(this.file); @override - Future runCustom(String statement) { - return _synchronized(() { - _logStmt(statement, const []); - _db.execute(statement); - }); - } + final TransactionDelegate transactionDelegate = const NoTransactionDelegate(); - void _runWithArgs(String statement, List args) { - _logStmt(statement, args); + @override + DbVersionDelegate versionDelegate; - if (args.isEmpty) { - _db.execute(statement); + @override + Future get isOpen => Future.value(_db != null); + + @override + Future open([GeneratedDatabase db]) { + if (file != null) { + _db = Database.openFile(file); } else { - _db.prepare(statement) - ..execute(args) - ..close(); + _db = Database.memory(); } - } - - Future _runAndReturnAffected(String statement, List args) { - _runWithArgs(statement, args); - return Future.value(_db.updatedRows); - } - - @override - Future runDelete(String statement, List args) { - return _synchronized(() { - return _runAndReturnAffected(statement, args); - }); - } - - @override - Future runUpdate(String statement, List args) { - return _synchronized(() { - return _runAndReturnAffected(statement, args); - }); - } - - @override - Future runInsert(String statement, List args) { - return _synchronized(() { - _runWithArgs(statement, args); - return Future.value(_db.lastInsertId); - }); - } - - @override - Future>> runSelect( - String statement, List args) { - return _synchronized(() { - _logStmt(statement, args); - final stmt = _db.prepare(statement); - final result = stmt.select(args); - stmt.close(); - - return Future.value(result.toList()); - }); + versionDelegate = _VmVersionDelegate(_db); + return Future.value(); } @override @@ -107,103 +58,56 @@ abstract class _DatabaseUser extends QueryExecutor { return Future.value(); } -} -class VMDatabase extends _DatabaseUser { - VMDatabase(File file, {bool logStatements = false}) - : super(_DbState(logStatements, file)); - - VMDatabase.memory({bool logStatements = false}) - : this(null, logStatements: logStatements); - - @override - Future ensureOpen() async { - if (_db == null) { - _state.db = _openInternal(); - await _runMigrations(); - } - return true; - } - - Database _openInternal() { - if (_state.file == null) { - return Database.memory(); + void _runWithArgs(String statement, List args) { + if (args.isEmpty) { + _db.execute(statement); } else { - return Database.openFile(_state.file); + _db.prepare(statement) + ..execute(args) + ..close(); } } - Future _runMigrations() async { - final current = _db.userVersion; - final target = databaseInfo.schemaVersion; - - if (current == 0) { - await databaseInfo.handleDatabaseCreation(executor: runCustom); - } else if (current < target) { - await databaseInfo.handleDatabaseVersionChange( - executor: null, from: current, to: target); - } - - _db.userVersion = target; - - await _synchronized(() { - databaseInfo.beforeOpenCallback( - _BeforeOpenExecutor(_state), OpeningDetails(current, target)); - }); - } - @override - Future close() { - _db?.close(); + Future runCustom(String statement, List args) { + _runWithArgs(statement, args); return Future.value(); } @override - TransactionExecutor beginTransaction() { - final transactionReady = Completer(); - final executor = _TransactionExecutor(_state, transactionReady.future); - - _synchronized(() async { - // we have the lock, so start the transaction - transactionReady.complete(true); - await executor.completed; - }); - - return executor; - } -} - -class _BeforeOpenExecutor extends _DatabaseUser with BeforeOpenMixin { - @override - final bool _bypassLock = true; - _BeforeOpenExecutor(_DbState state) : super(state); -} - -class _TransactionExecutor extends _DatabaseUser with TransactionExecutor { - @override - final bool _bypassLock = true; - final Future _openingFuture; - bool _sentBeginTransaction = false; - - final Completer _completer = Completer(); - Future get completed => _completer.future; - - _TransactionExecutor(_DbState state, this._openingFuture) : super(state); - - @override - Future ensureOpen() async { - await _openingFuture; - if (!_sentBeginTransaction) { - _db.execute('BEGIN TRANSACTION'); - _sentBeginTransaction = true; - } - return Future.value(true); + Future runInsert(String statement, List args) { + _runWithArgs(statement, args); + return Future.value(_db.lastInsertId); } @override - Future send() { - _db.execute('COMMIT TRANSACTION;'); - _completer.complete(); + Future runUpdate(String statement, List args) { + _runWithArgs(statement, args); + return Future.value(_db.updatedRows); + } + + @override + Future runSelect(String statement, List args) { + final stmt = _db.prepare(statement); + final result = stmt.select(args); + stmt.close(); + + return Future.value(QueryResult(result.columnNames, result.rows)); + } +} + +class _VmVersionDelegate extends DynamicVersionDelegate { + final Database database; + + _VmVersionDelegate(this.database); + + @override + Future get schemaVersion => Future.value(database.userVersion); + + @override + Future setSchemaVersion(int version) { + database.userVersion = version; return Future.value(); } } diff --git a/moor/lib/src/web/web_db.dart b/moor/lib/src/web/web_db.dart index 6e3dc966..f0c9b7e2 100644 --- a/moor/lib/src/web/web_db.dart +++ b/moor/lib/src/web/web_db.dart @@ -228,14 +228,23 @@ class WebDatabase extends _DatabaseUser { } } -class _BeforeOpenExecutor extends _DatabaseUser with BeforeOpenMixin { +class _BeforeOpenExecutor extends _DatabaseUser { _BeforeOpenExecutor(_DbState state) : super(state); @override final bool _bypassLock = true; + + @override + TransactionExecutor beginTransaction() { + throw Error(); + } + + @override + Future ensureOpen() => Future.value(true); } -class _TransactionExecutor extends _DatabaseUser with TransactionExecutor { +class _TransactionExecutor extends _DatabaseUser + implements TransactionExecutor { _TransactionExecutor(_DbState state, this._openingFuture) : super(state); @override @@ -270,4 +279,9 @@ class _TransactionExecutor extends _DatabaseUser with TransactionExecutor { _completer.complete(); return Future.value(); } + + @override + TransactionExecutor beginTransaction() { + throw Error(); + } } diff --git a/moor/test/vm/integration_test.dart b/moor/test/vm/integration_test.dart index 8adca5e1..f42d7131 100644 --- a/moor/test/vm/integration_test.dart +++ b/moor/test/vm/integration_test.dart @@ -1,4 +1,7 @@ +import 'dart:async'; + import 'package:moor/moor.dart'; +import 'package:pedantic/pedantic.dart'; import 'package:test_api/test_api.dart'; import 'package:moor/moor_vm.dart'; @@ -32,6 +35,29 @@ void main() { final readUpdated = await db.select(db.todosTable).getSingle(); expect(readUpdated.content, 'Updated content'); }); + + test('Transactions test', () async { + db = TodoDb(VMDatabase.memory(logStatements: false)); + + final completedOperations = StreamController(); + + unawaited(db.transaction((_) async { + await insertCategory(); + completedOperations.add('transaction'); + await pumpEventQueue(); + })); + + unawaited(insertUser().then((_) { + completedOperations.add('regular'); + })); + + await expectLater( + completedOperations.stream, emitsInOrder(['transaction', 'regular'])); + + // call .getSingle to verify both rows have been written + await db.select(db.users).getSingle(); + await db.select(db.categories).getSingle(); + }); } Future insertCategory() async { From 7ab63c0862778c6bcf52ee3c70d3a055a76e499a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 24 Jul 2019 22:21:52 +0200 Subject: [PATCH 008/117] Run the integration tests on the VM backend --- extras/integration_tests/vm/.gitignore | 11 ++++++++ extras/integration_tests/vm/pubspec.yaml | 15 +++++++++++ extras/integration_tests/vm/test/vm_test.dart | 27 +++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 extras/integration_tests/vm/.gitignore create mode 100644 extras/integration_tests/vm/pubspec.yaml create mode 100644 extras/integration_tests/vm/test/vm_test.dart diff --git a/extras/integration_tests/vm/.gitignore b/extras/integration_tests/vm/.gitignore new file mode 100644 index 00000000..50602ac6 --- /dev/null +++ b/extras/integration_tests/vm/.gitignore @@ -0,0 +1,11 @@ +# Files and directories created by pub +.dart_tool/ +.packages +# Remove the following pattern if you wish to check in your lock file +pubspec.lock + +# Conventional directory for build outputs +build/ + +# Directory created by dartdoc +doc/api/ diff --git a/extras/integration_tests/vm/pubspec.yaml b/extras/integration_tests/vm/pubspec.yaml new file mode 100644 index 00000000..fd4f4390 --- /dev/null +++ b/extras/integration_tests/vm/pubspec.yaml @@ -0,0 +1,15 @@ +name: vm +description: A sample command-line application. +# version: 1.0.0 +# homepage: https://www.example.com +# author: Simon Binder + +environment: + sdk: '>=2.4.0 <3.0.0' + +dependencies: + tests: + path: ../tests + +dev_dependencies: + test: ^1.5.0 diff --git a/extras/integration_tests/vm/test/vm_test.dart b/extras/integration_tests/vm/test/vm_test.dart new file mode 100644 index 00000000..a3fb763b --- /dev/null +++ b/extras/integration_tests/vm/test/vm_test.dart @@ -0,0 +1,27 @@ +import 'dart:io'; + +import 'package:moor/moor_vm.dart'; +import 'package:tests/tests.dart'; + +import 'package:path/path.dart' show join; + +class VmExecutor extends TestExecutor { + static String fileName = 'moor-vm-tests-${DateTime.now().toIso8601String()}'; + final file = File(join(Directory.systemTemp.path, fileName)); + + @override + QueryExecutor createExecutor() { + return VMDatabase(file); + } + + @override + Future deleteData() async { + if (await file.exists()) { + await file.delete(); + } + } +} + +void main() { + runAllTests(VmExecutor()); +} From 5ddcd17c21bc0ab70aeadaa39f12001bb0b7bd8a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 29 Jul 2019 10:27:50 +0200 Subject: [PATCH 009/117] Scaffold the package layout of a future analyzer plugin --- CONTRIBUTING.md | 3 +- moor/tool/analyzer_plugin/bin/plugin.dart | 7 ++++ moor/tool/analyzer_plugin/pubspec.yaml | 10 +++++ moor_generator/lib/plugin.dart | 1 + moor_generator/lib/src/plugin/driver.dart | 40 +++++++++++++++++++ moor_generator/lib/src/plugin/plugin.dart | 24 +++++++++++ moor_generator/lib/src/plugin/starter.dart | 10 +++++ .../lib/src/plugin/state/file_tracker.dart | 5 +++ moor_generator/pubspec.yaml | 1 + 9 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 moor/tool/analyzer_plugin/bin/plugin.dart create mode 100644 moor/tool/analyzer_plugin/pubspec.yaml create mode 100644 moor_generator/lib/plugin.dart create mode 100644 moor_generator/lib/src/plugin/driver.dart create mode 100644 moor_generator/lib/src/plugin/plugin.dart create mode 100644 moor_generator/lib/src/plugin/starter.dart create mode 100644 moor_generator/lib/src/plugin/state/file_tracker.dart diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a44bfb36..2c658435 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,8 @@ updates that span multiple versions, we should follow these steps 2. Make sure each package has the correct dependencies: `moor_flutter` version `1.x` should depend on moor `1.x` as well to ensure users will always `pub get` moor packages that are compatible with each other. -3. Comment out the `dependency_overrides` section +3. Comment out the `dependency_overrides` section in `moor`, `moor/tool/analyzer_plugin`, `moor_flutter`, + `moor_generator` and `sqlparser`. 4. Publish packages in this order to avoid scoring penalties caused by versions not existing: 1. `moor` 2. `moor_generator` diff --git a/moor/tool/analyzer_plugin/bin/plugin.dart b/moor/tool/analyzer_plugin/bin/plugin.dart new file mode 100644 index 00000000..61a56d45 --- /dev/null +++ b/moor/tool/analyzer_plugin/bin/plugin.dart @@ -0,0 +1,7 @@ +import 'dart:isolate'; + +import 'package:moor_generator/plugin.dart'; + +void main(List args, SendPort sendPort) { + start(args, sendPort); +} diff --git a/moor/tool/analyzer_plugin/pubspec.yaml b/moor/tool/analyzer_plugin/pubspec.yaml new file mode 100644 index 00000000..a3dacbcb --- /dev/null +++ b/moor/tool/analyzer_plugin/pubspec.yaml @@ -0,0 +1,10 @@ +name: analyzer_load_moor_plugin +version: 1.0.0 +description: This pubspec is a part of moor and determines the version of the moor analyzer to load + +dependencies: + moor_generator: ^1.6.0 + +dependency_overrides: + moor_generator: + path: ../../../moor_generator \ No newline at end of file diff --git a/moor_generator/lib/plugin.dart b/moor_generator/lib/plugin.dart new file mode 100644 index 00000000..6b21b8e9 --- /dev/null +++ b/moor_generator/lib/plugin.dart @@ -0,0 +1 @@ +export 'src/plugin/starter.dart'; diff --git a/moor_generator/lib/src/plugin/driver.dart b/moor_generator/lib/src/plugin/driver.dart new file mode 100644 index 00000000..b32e09f6 --- /dev/null +++ b/moor_generator/lib/src/plugin/driver.dart @@ -0,0 +1,40 @@ +// ignore_for_file: implementation_imports +import 'package:analyzer/src/dart/analysis/driver.dart'; + +class MoorDriver implements AnalysisDriverGeneric { + final _addedFiles = {}; + + bool _ownsFile(String path) => path.endsWith('.moor'); + + @override + void addFile(String path) { + if (_ownsFile(path)) { + _addedFiles.add(path); + handleFileChanged(path); + } + } + + @override + void dispose() {} + + void handleFileChanged(String path) { + if (_ownsFile(path)) {} + } + + @override + bool get hasFilesToAnalyze => null; + + @override + Future performWork() { + // TODO: implement performWork + return null; + } + + @override + set priorityFiles(List priorityPaths) { + // We don't support this ATM + } + + @override + AnalysisDriverPriority get workPriority => AnalysisDriverPriority.general; +} diff --git a/moor_generator/lib/src/plugin/plugin.dart b/moor_generator/lib/src/plugin/plugin.dart new file mode 100644 index 00000000..5a6505bb --- /dev/null +++ b/moor_generator/lib/src/plugin/plugin.dart @@ -0,0 +1,24 @@ +import 'package:analyzer/file_system/file_system.dart'; +// ignore: implementation_imports +import 'package:analyzer/src/dart/analysis/driver.dart'; +import 'package:analyzer_plugin/plugin/plugin.dart'; +import 'package:analyzer_plugin/protocol/protocol_generated.dart'; + +class MoorPlugin extends ServerPlugin { + MoorPlugin(ResourceProvider provider) : super(provider); + + @override + final List fileGlobsToAnalyze = const ['**/*.moor']; + @override + final String name = 'Moor plugin'; + @override + final String version = '0.0.1'; + @override + final String contactInfo = + 'Create an issue at https://github.com/simolus3/moor/'; + + @override + AnalysisDriverGeneric createAnalysisDriver(ContextRoot contextRoot) { + return null; + } +} diff --git a/moor_generator/lib/src/plugin/starter.dart b/moor_generator/lib/src/plugin/starter.dart new file mode 100644 index 00000000..815b31ce --- /dev/null +++ b/moor_generator/lib/src/plugin/starter.dart @@ -0,0 +1,10 @@ +import 'dart:isolate'; + +import 'package:analyzer/file_system/physical_file_system.dart'; +import 'package:analyzer_plugin/starter.dart'; +import 'package:moor_generator/src/plugin/plugin.dart'; + +void start(List args, SendPort sendPort) { + ServerPluginStarter(MoorPlugin(PhysicalResourceProvider.INSTANCE)) + .start(sendPort); +} diff --git a/moor_generator/lib/src/plugin/state/file_tracker.dart b/moor_generator/lib/src/plugin/state/file_tracker.dart new file mode 100644 index 00000000..bc19c46a --- /dev/null +++ b/moor_generator/lib/src/plugin/state/file_tracker.dart @@ -0,0 +1,5 @@ +/// Keeps track of files that need to be analyzed by the moor generator. +class FileTracker {} + +/// A `.moor` file added to the plugin. +class _TrackedFile {} diff --git a/moor_generator/pubspec.yaml b/moor_generator/pubspec.yaml index ff2799d9..bc1e16d4 100644 --- a/moor_generator/pubspec.yaml +++ b/moor_generator/pubspec.yaml @@ -13,6 +13,7 @@ environment: dependencies: analyzer: '>=0.36.0 <0.38.0' + analyzer_plugin: recase: ^2.0.1 built_value: '>=6.3.0 <7.0.0' source_gen: ^0.9.4 From 4210c0c836b00ffb4597dfa9b674a48d8392a3a0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 30 Jul 2019 18:34:33 +0200 Subject: [PATCH 010/117] Start some work on the analyzer plugin Of course it's not working at all --- analysis_options.yaml | 3 +- .../tests/analysis_options.yaml | 3 + .../tests/lib/database/test.moor | 3 + .../analyzer_plugin/bin/plugin.dart | 0 .../analyzer_plugin/pubspec.yaml | 10 +- .../lib/src/parser/moor/moor_analyzer.dart | 4 +- .../plugin/analyzer/highlights/request.dart | 14 ++ .../analyzer/highlights/sql_highlighter.dart | 82 +++++++++++ .../src/plugin/analyzer/moor_analyzer.dart | 15 ++ .../lib/src/plugin/analyzer/results.dart | 8 ++ moor_generator/lib/src/plugin/driver.dart | 59 ++++++-- moor_generator/lib/src/plugin/plugin.dart | 56 +++++++- .../lib/src/plugin/state/file_tracker.dart | 131 +++++++++++++++++- moor_generator/pubspec.yaml | 5 +- sqlparser/lib/sqlparser.dart | 2 +- sqlparser/lib/src/engine/sql_engine.dart | 12 +- 16 files changed, 371 insertions(+), 36 deletions(-) create mode 100644 extras/integration_tests/tests/analysis_options.yaml create mode 100644 extras/integration_tests/tests/lib/database/test.moor rename moor/{tool => tools}/analyzer_plugin/bin/plugin.dart (100%) rename moor/{tool => tools}/analyzer_plugin/pubspec.yaml (52%) create mode 100644 moor_generator/lib/src/plugin/analyzer/highlights/request.dart create mode 100644 moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart create mode 100644 moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart create mode 100644 moor_generator/lib/src/plugin/analyzer/results.dart diff --git a/analysis_options.yaml b/analysis_options.yaml index bb75690b..1793a810 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -28,7 +28,6 @@ linter: - await_only_futures - camel_case_types - cancel_subscriptions - - cascade_invocations - comment_references - constant_identifier_names - curly_braces_in_flow_control_structures @@ -83,4 +82,4 @@ linter: - unnecessary_this - unrelated_type_equality_checks - use_rethrow_when_possible - - valid_regexps + - valid_regexps \ No newline at end of file diff --git a/extras/integration_tests/tests/analysis_options.yaml b/extras/integration_tests/tests/analysis_options.yaml new file mode 100644 index 00000000..e649ddc0 --- /dev/null +++ b/extras/integration_tests/tests/analysis_options.yaml @@ -0,0 +1,3 @@ +analyzer: + plugins: + - moor \ No newline at end of file diff --git a/extras/integration_tests/tests/lib/database/test.moor b/extras/integration_tests/tests/lib/database/test.moor new file mode 100644 index 00000000..e2d524e3 --- /dev/null +++ b/extras/integration_tests/tests/lib/database/test.moor @@ -0,0 +1,3 @@ +CREATE TABLE test ( + id INT NOT NULL PRIMARY AUTOINCREMENT +) \ No newline at end of file diff --git a/moor/tool/analyzer_plugin/bin/plugin.dart b/moor/tools/analyzer_plugin/bin/plugin.dart similarity index 100% rename from moor/tool/analyzer_plugin/bin/plugin.dart rename to moor/tools/analyzer_plugin/bin/plugin.dart diff --git a/moor/tool/analyzer_plugin/pubspec.yaml b/moor/tools/analyzer_plugin/pubspec.yaml similarity index 52% rename from moor/tool/analyzer_plugin/pubspec.yaml rename to moor/tools/analyzer_plugin/pubspec.yaml index a3dacbcb..e66dd028 100644 --- a/moor/tool/analyzer_plugin/pubspec.yaml +++ b/moor/tools/analyzer_plugin/pubspec.yaml @@ -3,8 +3,10 @@ version: 1.0.0 description: This pubspec is a part of moor and determines the version of the moor analyzer to load dependencies: - moor_generator: ^1.6.0 - -dependency_overrides: moor_generator: - path: ../../../moor_generator \ No newline at end of file + +#dependency_overrides: +# moor_generator: +# path: /home/simon/IdeaProjects/moor/moor_generator +# sqlparser: +# path: /home/simon/IdeaProjects/moor/sqlparser \ No newline at end of file diff --git a/moor_generator/lib/src/parser/moor/moor_analyzer.dart b/moor_generator/lib/src/parser/moor/moor_analyzer.dart index 342c27b4..c0a93e7c 100644 --- a/moor_generator/lib/src/parser/moor/moor_analyzer.dart +++ b/moor_generator/lib/src/parser/moor/moor_analyzer.dart @@ -11,7 +11,9 @@ class MoorAnalyzer { MoorAnalyzer(this.content); Future analyze() { - final results = SqlEngine().parseMultiple(content); + final engine = SqlEngine(); + final tokens = engine.tokenize(content); + final results = SqlEngine().parseMultiple(tokens, content); final createdTables = []; final errors = []; diff --git a/moor_generator/lib/src/plugin/analyzer/highlights/request.dart b/moor_generator/lib/src/plugin/analyzer/highlights/request.dart new file mode 100644 index 00000000..65f9fbee --- /dev/null +++ b/moor_generator/lib/src/plugin/analyzer/highlights/request.dart @@ -0,0 +1,14 @@ +import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; + +import '../results.dart'; + +class MoorHighlightingRequest extends HighlightsRequest { + @override + final String path; + @override + final ResourceProvider resourceProvider; + final MoorAnalysisResults parsedFile; + + MoorHighlightingRequest(this.parsedFile, this.path, this.resourceProvider); +} diff --git a/moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart b/moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart new file mode 100644 index 00000000..4eb3f0b1 --- /dev/null +++ b/moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart @@ -0,0 +1,82 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; +import 'package:moor_generator/src/plugin/analyzer/highlights/request.dart'; +import 'package:sqlparser/sqlparser.dart'; + +const _notBuiltIn = { + TokenType.numberLiteral, + TokenType.stringLiteral, + TokenType.identifier, + TokenType.leftParen, + TokenType.rightParen, + TokenType.comma, + TokenType.star, + TokenType.less, + TokenType.lessEqual, + TokenType.lessMore, + TokenType.equal, + TokenType.more, + TokenType.moreEqual, + TokenType.shiftRight, + TokenType.shiftLeft, + TokenType.exclamationEqual, + TokenType.plus, + TokenType.minus, +}; + +class SqlHighlighter implements HighlightsContributor { + const SqlHighlighter(); + + @override + void computeHighlights( + HighlightsRequest request, HighlightsCollector collector) { + if (request is! MoorHighlightingRequest) { + return; + } + + final typedRequest = request as MoorHighlightingRequest; + final visitor = _HighlightingVisitor(collector); + + for (var stmt in typedRequest.parsedFile.statements) { + stmt.accept(visitor); + } + + for (var token in typedRequest.parsedFile.sqlTokens) { + if (!_notBuiltIn.contains(token.type)) { + final start = token.span.start.offset; + final length = token.span.length; + collector.addRegion(start, length, HighlightRegionType.BUILT_IN); + } + } + } +} + +class _HighlightingVisitor extends RecursiveVisitor { + final HighlightsCollector collector; + + _HighlightingVisitor(this.collector); + + void _contribute(AstNode node, HighlightRegionType type) { + final offset = node.firstPosition; + final length = node.lastPosition - offset; + collector.addRegion(offset, length, type); + } + + @override + void visitReference(Reference e) { + _contribute(e, HighlightRegionType.INSTANCE_FIELD_REFERENCE); + } + + @override + void visitLiteral(Literal e) { + if (e is NullLiteral) { + _contribute(e, HighlightRegionType.BUILT_IN); + } else if (e is NumericLiteral) { + _contribute(e, HighlightRegionType.LITERAL_INTEGER); + } else if (e is StringLiteral) { + _contribute(e, HighlightRegionType.LITERAL_STRING); + } else if (e is BooleanLiteral) { + _contribute(e, HighlightRegionType.LITERAL_BOOLEAN); + } + } +} diff --git a/moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart b/moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart new file mode 100644 index 00000000..a5456bfd --- /dev/null +++ b/moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart @@ -0,0 +1,15 @@ +import 'package:analyzer/file_system/file_system.dart'; +import 'package:moor_generator/src/plugin/analyzer/results.dart'; +import 'package:sqlparser/sqlparser.dart'; + +class MoorAnalyzer { + Future analyze(File file) async { + final content = file.readAsStringSync(); + final sqlEngine = SqlEngine(); + + final tokens = sqlEngine.tokenize(content); + final stmts = sqlEngine.parseMultiple(tokens, content); + + return MoorAnalysisResults(stmts.map((r) => r.rootNode).toList(), tokens); + } +} diff --git a/moor_generator/lib/src/plugin/analyzer/results.dart b/moor_generator/lib/src/plugin/analyzer/results.dart new file mode 100644 index 00000000..9aa4cc2b --- /dev/null +++ b/moor_generator/lib/src/plugin/analyzer/results.dart @@ -0,0 +1,8 @@ +import 'package:sqlparser/sqlparser.dart'; + +class MoorAnalysisResults { + final List statements; + final List sqlTokens; + + MoorAnalysisResults(this.statements, this.sqlTokens); +} diff --git a/moor_generator/lib/src/plugin/driver.dart b/moor_generator/lib/src/plugin/driver.dart index b32e09f6..3f16e40d 100644 --- a/moor_generator/lib/src/plugin/driver.dart +++ b/moor_generator/lib/src/plugin/driver.dart @@ -1,40 +1,79 @@ // ignore_for_file: implementation_imports +import 'dart:async'; + +import 'package:analyzer/file_system/file_system.dart'; import 'package:analyzer/src/dart/analysis/driver.dart'; +import 'package:moor_generator/src/plugin/state/file_tracker.dart'; + +import 'analyzer/moor_analyzer.dart'; +import 'analyzer/results.dart'; class MoorDriver implements AnalysisDriverGeneric { - final _addedFiles = {}; + final FileTracker _tracker; + final AnalysisDriverScheduler _scheduler; + final MoorAnalyzer _analyzer; + final ResourceProvider _resources; + + MoorDriver(this._tracker, this._scheduler, this._analyzer, this._resources) { + _scheduler.add(this); + } bool _ownsFile(String path) => path.endsWith('.moor'); @override void addFile(String path) { if (_ownsFile(path)) { - _addedFiles.add(path); - handleFileChanged(path); + _tracker.addFile(path); } } @override - void dispose() {} + void dispose() { + _scheduler.remove(this); + } void handleFileChanged(String path) { - if (_ownsFile(path)) {} + if (_ownsFile(path)) { + _tracker.handleContentChanged(path); + _scheduler.notify(this); + } } @override - bool get hasFilesToAnalyze => null; + bool get hasFilesToAnalyze => _tracker.hasWork; @override - Future performWork() { - // TODO: implement performWork - return null; + Future performWork() async { + final completer = Completer(); + + if (_tracker.hasWork) { + _tracker.work((path) { + try { + return _resolveMoorFile(path); + } finally { + completer.complete(); + } + }); + + await completer.future; + } + } + + Future _resolveMoorFile(String path) { + return _analyzer.analyze(_resources.getFile(path)); } @override set priorityFiles(List priorityPaths) { - // We don't support this ATM + _tracker.setPriorityFiles(priorityPaths); } @override + // todo ask the tracker about the top-priority file. AnalysisDriverPriority get workPriority => AnalysisDriverPriority.general; + + Future parseMoorFile(String path) { + _scheduler.notify(this); + return _tracker.results(path); + } } diff --git a/moor_generator/lib/src/plugin/plugin.dart b/moor_generator/lib/src/plugin/plugin.dart index 5a6505bb..3d6eacda 100644 --- a/moor_generator/lib/src/plugin/plugin.dart +++ b/moor_generator/lib/src/plugin/plugin.dart @@ -1,24 +1,66 @@ import 'package:analyzer/file_system/file_system.dart'; -// ignore: implementation_imports -import 'package:analyzer/src/dart/analysis/driver.dart'; +import 'package:analyzer_plugin/plugin/highlights_mixin.dart'; import 'package:analyzer_plugin/plugin/plugin.dart'; +import 'package:analyzer_plugin/protocol/protocol.dart'; import 'package:analyzer_plugin/protocol/protocol_generated.dart'; +import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; +import 'package:moor_generator/src/plugin/state/file_tracker.dart'; -class MoorPlugin extends ServerPlugin { +import 'analyzer/highlights/request.dart'; +import 'analyzer/highlights/sql_highlighter.dart'; +import 'analyzer/moor_analyzer.dart'; +import 'driver.dart'; + +class MoorPlugin extends ServerPlugin with HighlightsMixin { MoorPlugin(ResourceProvider provider) : super(provider); @override - final List fileGlobsToAnalyze = const ['**/*.moor']; + final List fileGlobsToAnalyze = const ['*.moor']; @override final String name = 'Moor plugin'; @override - final String version = '0.0.1'; + // docs say that this should a version of _this_ plugin, but they lie. this + // version will be used to determine compatibility with the analyzer + final String version = '2.0.0-alpha.0'; @override final String contactInfo = 'Create an issue at https://github.com/simolus3/moor/'; @override - AnalysisDriverGeneric createAnalysisDriver(ContextRoot contextRoot) { - return null; + MoorDriver createAnalysisDriver(ContextRoot contextRoot) { + final tracker = FileTracker(); + final analyzer = MoorAnalyzer(); + return MoorDriver( + tracker, analysisDriverScheduler, analyzer, resourceProvider); + } + + @override + void contentChanged(String path) { + _moorDriverForPath(path)?.handleFileChanged(path); + } + + MoorDriver _moorDriverForPath(String path) { + final driver = super.driverForPath(path); + + if (driver is! MoorDriver) return null; + return driver as MoorDriver; + } + + @override + List getHighlightsContributors(String path) { + return const [SqlHighlighter()]; + } + + @override + Future getHighlightsRequest(String path) async { + final driver = _moorDriverForPath(path); + if (driver == null) { + throw RequestFailure( + RequestErrorFactory.pluginError('Not driver set for path', null)); + } + + final parsed = await driver.parseMoorFile(path); + + return MoorHighlightingRequest(parsed, path, resourceProvider); } } diff --git a/moor_generator/lib/src/plugin/state/file_tracker.dart b/moor_generator/lib/src/plugin/state/file_tracker.dart index bc19c46a..a77fdc58 100644 --- a/moor_generator/lib/src/plugin/state/file_tracker.dart +++ b/moor_generator/lib/src/plugin/state/file_tracker.dart @@ -1,5 +1,128 @@ -/// Keeps track of files that need to be analyzed by the moor generator. -class FileTracker {} +import 'dart:async'; -/// A `.moor` file added to the plugin. -class _TrackedFile {} +import 'package:collection/collection.dart'; +import 'package:moor_generator/src/plugin/analyzer/results.dart'; + +/// Keeps track of files that need to be analyzed by the moor plugin. +class FileTracker { + PriorityQueue _pendingWork; + final Map _trackedFiles = {}; + final Set _currentPriority = {}; + + FileTracker() { + _pendingWork = PriorityQueue(_compareByPriority); + } + + int _compareByPriority(TrackedFile a, TrackedFile b) { + final aPriority = a.currentPriority?.index ?? 0; + final bPriority = b.currentPriority?.index ?? 0; + return aPriority.compareTo(bPriority); + } + + void _updateFile(TrackedFile file, Function(TrackedFile) update) { + _pendingWork.remove(file); + update(file); + _pendingWork.add(file); + } + + void _putInQueue(TrackedFile file) { + _updateFile(file, (f) { + // no action needed, insert with current priority. + }); + } + + bool get hasWork => _pendingWork.isNotEmpty; + + TrackedFile addFile(String path) { + return _trackedFiles.putIfAbsent(path, () { + final tracked = TrackedFile(path); + _pendingWork.add(tracked); + return tracked; + }); + } + + void handleContentChanged(String path) { + _putInQueue(addFile(path)); + } + + void setPriorityFiles(List priority) { + // remove prioritized flag from existing files + for (var file in _currentPriority) { + _updateFile(file, (f) => f._prioritized = false); + } + _currentPriority + ..clear() + ..addAll(priority.map(addFile)) + ..forEach((file) { + _updateFile(file, (f) => f._prioritized = true); + }); + } + + void notifyFileChanged(String path) { + final tracked = addFile(path); + tracked._currentResult = null; + _putInQueue(tracked); + } + + Future results(String path) async { + final tracked = addFile(path); + + if (tracked._currentResult != null) { + return tracked._currentResult; + } else { + final completer = Completer(); + tracked._waiting.add(completer); + return completer.future; + } + } + + void work(Future Function(String path) worker) { + if (_pendingWork.isNotEmpty) { + final unit = _pendingWork.removeFirst(); + + worker(unit.path).then((result) { + for (var completer in unit._waiting) { + completer.complete(result); + } + unit._waiting.clear(); + }, onError: (e, StackTrace s) { + for (var completer in unit._waiting) { + completer.completeError(e, s); + } + unit._waiting.clear(); + }); + } + } +} + +enum FileType { moor, unknown } + +enum FilePriority { ignore, regular, interactive } + +const Map _defaultPrio = { + FileType.moor: FilePriority.regular, + FileType.unknown: FilePriority.ignore, +}; + +class TrackedFile { + final String path; + final FileType type; + + /// Whether this file has been given an elevated priority, for instance + /// because the user is currently typing in it. + bool _prioritized; + MoorAnalysisResults _currentResult; + final List> _waiting = []; + + FilePriority get currentPriority => + _prioritized ? FilePriority.interactive : defaultPriority; + + TrackedFile._(this.path, this.type); + + factory TrackedFile(String path) { + final type = path.endsWith('.moor') ? FileType.moor : FileType.unknown; + return TrackedFile._(path, type); + } + + FilePriority get defaultPriority => _defaultPrio[type]; +} diff --git a/moor_generator/pubspec.yaml b/moor_generator/pubspec.yaml index bc1e16d4..31a48e79 100644 --- a/moor_generator/pubspec.yaml +++ b/moor_generator/pubspec.yaml @@ -14,8 +14,9 @@ environment: dependencies: analyzer: '>=0.36.0 <0.38.0' analyzer_plugin: + collection: ^1.14.0 recase: ^2.0.1 - built_value: '>=6.3.0 <7.0.0' + built_value: ^6.3.0 source_gen: ^0.9.4 source_span: ^1.5.5 build: ^1.1.0 @@ -28,7 +29,7 @@ dev_dependencies: test_api: ^0.2.0 test_core: ^0.2.0 build_runner: '>=1.1.0 <1.6.0' - built_value_generator: '>=6.3.0 <7.0.0' + built_value_generator: ^6.3.0 build_test: '>=0.10.0 <0.11.0' dependency_overrides: diff --git a/sqlparser/lib/sqlparser.dart b/sqlparser/lib/sqlparser.dart index 21cd4a51..5de3d55a 100644 --- a/sqlparser/lib/sqlparser.dart +++ b/sqlparser/lib/sqlparser.dart @@ -5,4 +5,4 @@ export 'src/analysis/analysis.dart'; export 'src/ast/ast.dart'; export 'src/engine/sql_engine.dart'; export 'src/reader/parser/parser.dart' show ParsingError; -export 'src/reader/tokenizer/token.dart' show CumulatedTokenizerException; +export 'src/reader/tokenizer/token.dart' hide keywords; diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index bdda0f2c..6b2211cf 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -44,14 +44,13 @@ class SqlEngine { final parser = Parser(tokens); final stmt = parser.statement(); - return ParseResult._(stmt, parser.errors, sql); + return ParseResult._(stmt, tokens, parser.errors, sql); } /// Parses multiple sql statements, separated by a semicolon. All /// [ParseResult] entries will have the same [ParseResult.errors], but the /// [ParseResult.sql] will only refer to the substring creating a statement. - List parseMultiple(String sql) { - final tokens = tokenize(sql); + List parseMultiple(List tokens, String sql) { final parser = Parser(tokens); final stmts = parser.statements(); @@ -61,7 +60,7 @@ class SqlEngine { final last = statement.lastPosition; final source = sql.substring(first, last); - return ParseResult._(statement, parser.errors, source); + return ParseResult._(statement, tokens, parser.errors, source); }).toList(); } @@ -113,6 +112,9 @@ class ParseResult { /// The topmost node in the sql AST that was parsed. final AstNode rootNode; + /// The tokens that were scanned in the source file + final List tokens; + /// A list of all errors that occurred during parsing. [ParsingError.toString] /// returns a helpful description of what went wrong, along with the position /// where the error occurred. @@ -121,5 +123,5 @@ class ParseResult { /// The sql source that created the AST at [rootNode]. final String sql; - ParseResult._(this.rootNode, this.errors, this.sql); + ParseResult._(this.rootNode, this.tokens, this.errors, this.sql); } From f4bd6828a9265abf71b162f6acfd8d3db8b0f482 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 31 Jul 2019 18:05:54 +0200 Subject: [PATCH 011/117] Some more fixes on the analyzer Sadly, the analyzer doesn't support custom file types yet, so that's it for the moor analyzer plugin for now. --- moor/tools/analyzer_plugin/pubspec.yaml | 5 ++++- moor_generator/lib/src/plugin/state/file_tracker.dart | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/moor/tools/analyzer_plugin/pubspec.yaml b/moor/tools/analyzer_plugin/pubspec.yaml index e66dd028..194c4bbf 100644 --- a/moor/tools/analyzer_plugin/pubspec.yaml +++ b/moor/tools/analyzer_plugin/pubspec.yaml @@ -5,8 +5,11 @@ description: This pubspec is a part of moor and determines the version of the mo dependencies: moor_generator: +# To work on this plugin, you need to add the absolute paths here. Relative paths aren't supported yet +# https://github.com/dart-lang/sdk/issues/35281 + #dependency_overrides: # moor_generator: # path: /home/simon/IdeaProjects/moor/moor_generator # sqlparser: -# path: /home/simon/IdeaProjects/moor/sqlparser \ No newline at end of file +# path: /home/simon/IdeaProjects/moor/sqlparser diff --git a/moor_generator/lib/src/plugin/state/file_tracker.dart b/moor_generator/lib/src/plugin/state/file_tracker.dart index a77fdc58..42c982b2 100644 --- a/moor_generator/lib/src/plugin/state/file_tracker.dart +++ b/moor_generator/lib/src/plugin/state/file_tracker.dart @@ -110,7 +110,7 @@ class TrackedFile { /// Whether this file has been given an elevated priority, for instance /// because the user is currently typing in it. - bool _prioritized; + bool _prioritized = false; MoorAnalysisResults _currentResult; final List> _waiting = []; From e38719a1946faebb59da3d3be3001d2c0f287649 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 Aug 2019 12:27:39 +0200 Subject: [PATCH 012/117] Write changelogs, prepare release 1.7 --- .../tests/lib/database/database.dart | 5 +++-- moor/CHANGELOG.md | 2 +- moor/pubspec.yaml | 6 +++--- moor_flutter/CHANGELOG.md | 16 ++++++++++++++++ moor_flutter/pubspec.yaml | 8 ++++---- moor_generator/CHANGELOG.md | 4 ++++ moor_generator/pubspec.yaml | 14 +++++++------- sqlparser/CHANGELOG.md | 4 ++++ sqlparser/pubspec.yaml | 2 +- 9 files changed, 43 insertions(+), 18 deletions(-) diff --git a/extras/integration_tests/tests/lib/database/database.dart b/extras/integration_tests/tests/lib/database/database.dart index e53263ef..c4590641 100644 --- a/extras/integration_tests/tests/lib/database/database.dart +++ b/extras/integration_tests/tests/lib/database/database.dart @@ -110,8 +110,9 @@ class Database extends _$Database { Future deleteUser(User user, {bool fail = false}) { return transaction((_) async { final id = user.id; - delete(friendships) - .where((f) => or(f.firstUser.equals(id), f.secondUser.equals(id))); + await (delete(friendships) + ..where((f) => or(f.firstUser.equals(id), f.secondUser.equals(id)))) + .go(); if (fail) { throw Exception('oh no, the query misteriously failed!'); diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 277bb79a..629372ff 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,4 +1,4 @@ -## unreleased +## 1.7.0 - Support custom columns via type converters. See the [docs](https://moor.simonbinder.eu/type_converters) for details on how to use this feature. - Transactions now roll back when not completed successfully, they also rethrow the exception diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index bea04ddc..165ae7c2 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -1,6 +1,6 @@ name: moor description: Moor is a safe and reactive persistence library for Dart applications -version: 1.6.0 +version: 1.7.0 repository: https://github.com/simolus3/moor homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues @@ -12,8 +12,8 @@ environment: sdk: '>=2.2.2 <3.0.0' dependencies: - meta: '>= 1.0.0 <2.0.0' - collection: '>= 1.0.0 <2.0.0' + meta: ^1.0.0 + collection: ^1.0.0 synchronized: ^2.1.0 pedantic: any diff --git a/moor_flutter/CHANGELOG.md b/moor_flutter/CHANGELOG.md index dceffae6..04986155 100644 --- a/moor_flutter/CHANGELOG.md +++ b/moor_flutter/CHANGELOG.md @@ -1,3 +1,19 @@ +## 1.7.0 +- Support custom columns via type converters. See the [docs](https://moor.simonbinder.eu/type_converters) +for details on how to use this feature. +- Transactions now roll back when not completed successfully, they also rethrow the exception +to make debugging easier. +- New `backends` api, making it easier to write database drivers that work with moor. Apart from +`moor_flutter`, new experimental backends can be checked out from git: + 1. `encrypted_moor`: An encrypted moor database: https://github.com/simolus3/moor/tree/develop/extras/encryption + 2. `moor_mysql`: Work in progress mysql backend for moor. https://github.com/simolus3/moor/tree/develop/extras/mysql +- The compiled sql feature is no longer experimental and will stay stable until a major version bump +- New, experimental support for `.moor` files! Instead of declaring your tables in Dart, you can + choose to declare them with sql by writing the `CREATE TABLE` statement in a `.moor` file. + You can then use these tables in the database and with daos by using the `include` parameter + on `@UseMoor` and `@UseDao`. Again, please notice that this is an experimental api and there + might be some hiccups. Please report any issues you run into. + ## 1.6.0 - Experimental web support! See [the documentation](https://moor.simonbinder.eu/web) for details. - Make transactions easier to use: Thanks to some Dart async magic, you no longer need to run diff --git a/moor_flutter/pubspec.yaml b/moor_flutter/pubspec.yaml index 3fc53ed1..9b55aba3 100644 --- a/moor_flutter/pubspec.yaml +++ b/moor_flutter/pubspec.yaml @@ -1,6 +1,6 @@ name: moor_flutter description: Flutter implementation of moor, a safe and reactive persistence library for Dart applications -version: 1.6.0 +version: 1.7.0 repository: https://github.com/simolus3/moor homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues @@ -12,10 +12,10 @@ environment: sdk: ">=2.0.0-dev.68.0 <3.0.0" dependencies: - moor: ^1.6.0 + moor: ^1.7.0 sqflite: ^1.1.0 - meta: '>=1.0.0 <1.2.0' - path: '>=1.0.0 <2.0.0' + meta: ^1.0.0 + path: ^1.0.0 flutter: sdk: flutter diff --git a/moor_generator/CHANGELOG.md b/moor_generator/CHANGELOG.md index 97e63add..733077f5 100644 --- a/moor_generator/CHANGELOG.md +++ b/moor_generator/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.7.0 +- Support type converters that were introduced in moor 1.7 +- Support parsing and generating code for `.moor` files (see [docs](https://moor.simonbinder.eu/docs/using-sql/custom_tables/)). + ## 1.6.0+2 - Generate code to expand array variables diff --git a/moor_generator/pubspec.yaml b/moor_generator/pubspec.yaml index 1c774a7e..3417e4f5 100644 --- a/moor_generator/pubspec.yaml +++ b/moor_generator/pubspec.yaml @@ -1,6 +1,6 @@ name: moor_generator description: Dev-dependency to generate table and dataclasses together with the moor package. -version: 1.6.0+2 +version: 1.7.0 repository: https://github.com/simolus3/moor homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues @@ -14,21 +14,21 @@ environment: dependencies: analyzer: '>=0.36.0 <0.39.0' recase: ^2.0.1 - built_value: '>=6.3.0 <7.0.0' + built_value: ^6.3.0 source_gen: ^0.9.4 source_span: ^1.5.5 build: ^1.1.0 build_config: '>=0.3.1 <1.0.0' - moor: ^1.6.0 - meta: '>= 1.0.0 <2.0.0' - sqlparser: ^0.1.2 + moor: ^1.7.0 + meta: ^1.1.0 + sqlparser: ^0.2.0 dev_dependencies: test: ^1.6.0 test_api: ^0.2.0 test_core: ^0.2.0 - build_runner: '>=1.1.0 <1.6.0' + build_runner: ^1.6.7 built_value_generator: '>=6.3.0 <7.0.0' - build_test: '>=0.10.0 <0.11.0' + build_test: ^0.10.0 dependency_overrides: sqlparser: diff --git a/sqlparser/CHANGELOG.md b/sqlparser/CHANGELOG.md index c48db485..961eb570 100644 --- a/sqlparser/CHANGELOG.md +++ b/sqlparser/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.2.0 +- Parse `CREATE TABLE` statements +- Extract schema information from parsed create table statements with `SchemaFromCreateTable`. + ## 0.1.2 - parse `COLLATE` expressions - fix wrong order in parsed `LIMIT` clauses diff --git a/sqlparser/pubspec.yaml b/sqlparser/pubspec.yaml index 60d45c55..9ff135fe 100644 --- a/sqlparser/pubspec.yaml +++ b/sqlparser/pubspec.yaml @@ -1,6 +1,6 @@ name: sqlparser description: Parses sqlite statements and performs static analysis on them -version: 0.1.2 +version: 0.2.0 homepage: https://github.com/simolus3/moor/tree/develop/sqlparser #homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues From 3e290188dc8c1319510f74dcbe90409b7ca44331 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 22 Aug 2019 13:46:43 +0200 Subject: [PATCH 013/117] Use explicit pedantic dependency in moor --- moor/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index 165ae7c2..179b6f48 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -15,7 +15,7 @@ dependencies: meta: ^1.0.0 collection: ^1.0.0 synchronized: ^2.1.0 - pedantic: any + pedantic: ^1.0.0 dev_dependencies: moor_generator: ^1.6.0 From 8bbfea305cf41fba7eec9aecb5bf6c5d0d03fcef Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 23 Aug 2019 22:03:49 +0200 Subject: [PATCH 014/117] Drop support for analyzer versions <0.37.0 We use MethodInvocation#typeArgumentTypes, which was introduced in version 0.37.0 --- moor_generator/CHANGELOG.md | 4 ++++ moor_generator/pubspec.yaml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moor_generator/CHANGELOG.md b/moor_generator/CHANGELOG.md index 733077f5..3008ebb3 100644 --- a/moor_generator/CHANGELOG.md +++ b/moor_generator/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.7.1 +- Drop support for analyzer versions `<0.36.4`. They weren't supported in version 1.7.0 either, but + the `pubspec.yaml` did not specify this correctly. + ## 1.7.0 - Support type converters that were introduced in moor 1.7 - Support parsing and generating code for `.moor` files (see [docs](https://moor.simonbinder.eu/docs/using-sql/custom_tables/)). diff --git a/moor_generator/pubspec.yaml b/moor_generator/pubspec.yaml index 3417e4f5..84a87a13 100644 --- a/moor_generator/pubspec.yaml +++ b/moor_generator/pubspec.yaml @@ -1,6 +1,6 @@ name: moor_generator description: Dev-dependency to generate table and dataclasses together with the moor package. -version: 1.7.0 +version: 1.7.1 repository: https://github.com/simolus3/moor homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues @@ -12,7 +12,7 @@ environment: sdk: '>=2.2.0 <3.0.0' dependencies: - analyzer: '>=0.36.0 <0.39.0' + analyzer: '>=0.36.4 <0.39.0' recase: ^2.0.1 built_value: ^6.3.0 source_gen: ^0.9.4 From 397d3f23836b5e6770126f40db66ad0c5456dcaa Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 23 Aug 2019 22:50:15 +0200 Subject: [PATCH 015/117] Support absent values on INTEGER PRIMARY KEY --- moor/example/example.g.dart | 9 +- moor/lib/src/runtime/structure/columns.dart | 24 ++- moor/test/data/tables/custom_tables.g.dart | 160 +++++++++++++++++- moor/test/data/tables/tables.moor | 9 +- moor/test/data/tables/todos.g.dart | 12 +- moor/test/schema_test.dart | 11 +- .../lib/src/model/specified_column.dart | 5 + .../lib/src/parser/column_parser.dart | 2 + .../lib/src/parser/moor/parsed_moor_file.dart | 1 + .../lib/src/writer/table_writer.dart | 5 + 10 files changed, 218 insertions(+), 20 deletions(-) diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index 0d9f098c..cdf8da0b 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -94,7 +94,8 @@ class $CategoriesTable extends Categories @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _descriptionMeta = @@ -288,7 +289,8 @@ class $RecipesTable extends Recipes with TableInfo<$RecipesTable, Recipe> { @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _titleMeta = const VerificationMeta('title'); @@ -500,7 +502,8 @@ class $IngredientsTable extends Ingredients @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _nameMeta = const VerificationMeta('name'); diff --git a/moor/lib/src/runtime/structure/columns.dart b/moor/lib/src/runtime/structure/columns.dart index 4ad576fb..02055d7d 100644 --- a/moor/lib/src/runtime/structure/columns.dart +++ b/moor/lib/src/runtime/structure/columns.dart @@ -174,6 +174,17 @@ class GeneratedBoolColumn extends GeneratedColumn class GeneratedIntColumn extends GeneratedColumn with ComparableExpr implements IntColumn { + /// Whether this column was declared to be a primary key via a column + /// constraint. The only way to do this in Dart is with + /// [IntColumnBuilder.autoIncrement]. In `.moor` files, declaring a column + /// to be `INTEGER NOT NULL PRIMARY KEY` will set this flag but not + /// [hasAutoIncrement]. If either field is enabled, this column will be an + /// alias for the rowid. + final bool declaredAsPrimaryKey; + + /// Whether this column was declared to be an `AUTOINCREMENT` column, either + /// with [IntColumnBuilder.autoIncrement] or with an `AUTOINCREMENT` clause + /// in a `.moor` file. final bool hasAutoIncrement; @override @@ -183,6 +194,7 @@ class GeneratedIntColumn extends GeneratedColumn String name, String tableName, bool nullable, { + this.declaredAsPrimaryKey = false, this.hasAutoIncrement = false, String $customConstraints, Expression defaultValue, @@ -190,18 +202,18 @@ class GeneratedIntColumn extends GeneratedColumn $customConstraints: $customConstraints, defaultValue: defaultValue); @override - void writeColumnDefinition(GenerationContext into) { - // todo make this work with custom constraints, default values, etc. + void writeCustomConstraints(StringBuffer into) { if (hasAutoIncrement) { - into.buffer.write('${$name} $typeName PRIMARY KEY AUTOINCREMENT'); - } else { - super.writeColumnDefinition(into); + into.write(' PRIMARY KEY AUTOINCREMENT'); + } else if (declaredAsPrimaryKey) { + into.write('PRIMARY KEY'); } } @override bool get isRequired { - return !hasAutoIncrement && super.isRequired; + final aliasForRowId = declaredAsPrimaryKey || hasAutoIncrement; + return !aliasForRowId && super.isRequired; } } diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index e839775f..bdf39b2e 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -622,6 +622,162 @@ class Config extends Table with TableInfo { final bool dontWriteConstraints = true; } +class MytableData extends DataClass implements Insertable { + final int someid; + final String sometext; + MytableData({@required this.someid, this.sometext}); + factory MytableData.fromData(Map data, GeneratedDatabase db, + {String prefix}) { + final effectivePrefix = prefix ?? ''; + final intType = db.typeSystem.forDartType(); + final stringType = db.typeSystem.forDartType(); + return MytableData( + someid: intType.mapFromDatabaseResponse(data['${effectivePrefix}someid']), + sometext: stringType + .mapFromDatabaseResponse(data['${effectivePrefix}sometext']), + ); + } + factory MytableData.fromJson(Map json, + {ValueSerializer serializer = const ValueSerializer.defaults()}) { + return MytableData( + someid: serializer.fromJson(json['someid']), + sometext: serializer.fromJson(json['sometext']), + ); + } + @override + Map toJson( + {ValueSerializer serializer = const ValueSerializer.defaults()}) { + return { + 'someid': serializer.toJson(someid), + 'sometext': serializer.toJson(sometext), + }; + } + + @override + T createCompanion>(bool nullToAbsent) { + return MytableCompanion( + someid: + someid == null && nullToAbsent ? const Value.absent() : Value(someid), + sometext: sometext == null && nullToAbsent + ? const Value.absent() + : Value(sometext), + ) as T; + } + + MytableData copyWith({int someid, String sometext}) => MytableData( + someid: someid ?? this.someid, + sometext: sometext ?? this.sometext, + ); + @override + String toString() { + return (StringBuffer('MytableData(') + ..write('someid: $someid, ') + ..write('sometext: $sometext') + ..write(')')) + .toString(); + } + + @override + int get hashCode => $mrjf($mrjc(someid.hashCode, sometext.hashCode)); + @override + bool operator ==(other) => + identical(this, other) || + (other is MytableData && + other.someid == someid && + other.sometext == sometext); +} + +class MytableCompanion extends UpdateCompanion { + final Value someid; + final Value sometext; + const MytableCompanion({ + this.someid = const Value.absent(), + this.sometext = const Value.absent(), + }); + MytableCompanion copyWith({Value someid, Value sometext}) { + return MytableCompanion( + someid: someid ?? this.someid, + sometext: sometext ?? this.sometext, + ); + } +} + +class Mytable extends Table with TableInfo { + final GeneratedDatabase _db; + final String _alias; + Mytable(this._db, [this._alias]); + final VerificationMeta _someidMeta = const VerificationMeta('someid'); + GeneratedIntColumn _someid; + GeneratedIntColumn get someid => _someid ??= _constructSomeid(); + GeneratedIntColumn _constructSomeid() { + return GeneratedIntColumn('someid', $tableName, false, + declaredAsPrimaryKey: true, $customConstraints: 'NOT NULL PRIMARY KEY'); + } + + final VerificationMeta _sometextMeta = const VerificationMeta('sometext'); + GeneratedTextColumn _sometext; + GeneratedTextColumn get sometext => _sometext ??= _constructSometext(); + GeneratedTextColumn _constructSometext() { + return GeneratedTextColumn('sometext', $tableName, true, + $customConstraints: ''); + } + + @override + List get $columns => [someid, sometext]; + @override + Mytable get asDslTable => this; + @override + String get $tableName => _alias ?? 'mytable'; + @override + final String actualTableName = 'mytable'; + @override + VerificationContext validateIntegrity(MytableCompanion d, + {bool isInserting = false}) { + final context = VerificationContext(); + if (d.someid.present) { + context.handle( + _someidMeta, someid.isAcceptableValue(d.someid.value, _someidMeta)); + } else if (someid.isRequired && isInserting) { + context.missing(_someidMeta); + } + if (d.sometext.present) { + context.handle(_sometextMeta, + sometext.isAcceptableValue(d.sometext.value, _sometextMeta)); + } else if (sometext.isRequired && isInserting) { + context.missing(_sometextMeta); + } + return context; + } + + @override + Set get $primaryKey => {someid}; + @override + MytableData map(Map data, {String tablePrefix}) { + final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : null; + return MytableData.fromData(data, _db, prefix: effectivePrefix); + } + + @override + Map entityToSql(MytableCompanion d) { + final map = {}; + if (d.someid.present) { + map['someid'] = Variable(d.someid.value); + } + if (d.sometext.present) { + map['sometext'] = Variable(d.sometext.value); + } + return map; + } + + @override + Mytable createAlias(String alias) { + return Mytable(_db, alias); + } + + @override + final bool dontWriteConstraints = true; +} + abstract class _$CustomTablesDb extends GeneratedDatabase { _$CustomTablesDb(QueryExecutor e) : super(const SqlTypeSystem.withDefaults(), e); @@ -634,7 +790,9 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { _withConstraints ??= WithConstraints(this); Config _config; Config get config => _config ??= Config(this); + Mytable _mytable; + Mytable get mytable => _mytable ??= Mytable(this); @override List get allTables => - [noIds, withDefaults, withConstraints, config]; + [noIds, withDefaults, withConstraints, config, mytable]; } diff --git a/moor/test/data/tables/tables.moor b/moor/test/data/tables/tables.moor index 87bf471e..a36c1b09 100644 --- a/moor/test/data/tables/tables.moor +++ b/moor/test/data/tables/tables.moor @@ -5,7 +5,7 @@ CREATE TABLE no_ids ( CREATE TABLE with_defaults ( a TEXT DEFAULT 'something', b INT UNIQUE -) +); CREATE TABLE with_constraints ( a TEXT, @@ -13,9 +13,14 @@ CREATE TABLE with_constraints ( c FLOAT(10, 2), FOREIGN KEY (a, b) REFERENCES with_defaults (a, b) -) +); create table config ( config_key TEXT not null primary key, config_value TEXT ); + +CREATE TABLE mytable ( + someid INTEGER NOT NULL PRIMARY KEY, + sometext TEXT +) \ No newline at end of file diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 87dd1801..0caf9e30 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -159,7 +159,8 @@ class $TodosTableTable extends TodosTable @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _titleMeta = const VerificationMeta('title'); @@ -375,7 +376,8 @@ class $CategoriesTable extends Categories @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _descriptionMeta = @@ -592,7 +594,8 @@ class $UsersTable extends Users with TableInfo<$UsersTable, User> { @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _nameMeta = const VerificationMeta('name'); @@ -1177,7 +1180,8 @@ class $PureDefaultsTable extends PureDefaults @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _txtMeta = const VerificationMeta('txt'); diff --git a/moor/test/schema_test.dart b/moor/test/schema_test.dart index d6ca3a07..645d6297 100644 --- a/moor/test/schema_test.dart +++ b/moor/test/schema_test.dart @@ -22,19 +22,21 @@ void main() { // should create todos, categories, users and shared_todos table verify(mockQueryExecutor.call( 'CREATE TABLE IF NOT EXISTS todos ' - '(id INTEGER PRIMARY KEY AUTOINCREMENT, title VARCHAR NULL, ' + '(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, title VARCHAR NULL, ' 'content VARCHAR NOT NULL, target_date INTEGER NULL, ' 'category INTEGER NULL);', [])); verify(mockQueryExecutor.call( 'CREATE TABLE IF NOT EXISTS categories ' - '(id INTEGER PRIMARY KEY AUTOINCREMENT, `desc` VARCHAR NOT NULL UNIQUE);', + '(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, ' + '`desc` VARCHAR NOT NULL UNIQUE);', [])); verify(mockQueryExecutor.call( 'CREATE TABLE IF NOT EXISTS users ' - '(id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR NOT NULL, ' + '(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, ' + 'name VARCHAR NOT NULL, ' 'is_awesome BOOLEAN NOT NULL DEFAULT 1 CHECK (is_awesome in (0, 1)), ' 'profile_picture BLOB NOT NULL, ' 'creation_time INTEGER NOT NULL ' @@ -66,7 +68,8 @@ void main() { verify(mockQueryExecutor.call( 'CREATE TABLE IF NOT EXISTS users ' - '(id INTEGER PRIMARY KEY AUTOINCREMENT, name VARCHAR NOT NULL, ' + '(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, ' + 'name VARCHAR NOT NULL, ' 'is_awesome BOOLEAN NOT NULL DEFAULT 1 CHECK (is_awesome in (0, 1)), ' 'profile_picture BLOB NOT NULL, ' 'creation_time INTEGER NOT NULL ' diff --git a/moor_generator/lib/src/model/specified_column.dart b/moor_generator/lib/src/model/specified_column.dart index 913781ac..75d59b2e 100644 --- a/moor_generator/lib/src/model/specified_column.dart +++ b/moor_generator/lib/src/model/specified_column.dart @@ -164,6 +164,11 @@ abstract class ColumnFeature { const ColumnFeature(); } +/// A `PRIMARY KEY` column constraint. +class PrimaryKey extends ColumnFeature { + const PrimaryKey(); +} + class AutoIncrement extends ColumnFeature { static const AutoIncrement _instance = AutoIncrement._(); diff --git a/moor_generator/lib/src/parser/column_parser.dart b/moor_generator/lib/src/parser/column_parser.dart index 412c4929..a02ddc65 100644 --- a/moor_generator/lib/src/parser/column_parser.dart +++ b/moor_generator/lib/src/parser/column_parser.dart @@ -126,6 +126,8 @@ class ColumnParser extends ParserBase { break; case _methodAutoIncrement: foundFeatures.add(AutoIncrement()); + // a column declared as auto increment is always a primary key + foundFeatures.add(const PrimaryKey()); break; case _methodNullable: nullable = true; diff --git a/moor_generator/lib/src/parser/moor/parsed_moor_file.dart b/moor_generator/lib/src/parser/moor/parsed_moor_file.dart index a91fd353..b9f712cf 100644 --- a/moor_generator/lib/src/parser/moor/parsed_moor_file.dart +++ b/moor_generator/lib/src/parser/moor/parsed_moor_file.dart @@ -52,6 +52,7 @@ class CreateTable { for (var constraint in column.constraints) { if (constraint is PrimaryKeyColumn) { isPrimaryKey = true; + features.add(const PrimaryKey()); if (constraint.autoIncrement) { features.add(AutoIncrement()); } diff --git a/moor_generator/lib/src/writer/table_writer.dart b/moor_generator/lib/src/writer/table_writer.dart index 79d3eba1..05996443 100644 --- a/moor_generator/lib/src/writer/table_writer.dart +++ b/moor_generator/lib/src/writer/table_writer.dart @@ -140,6 +140,11 @@ class TableWriter { if (feature.maxLength != null) { additionalParams['maxTextLength'] = feature.maxLength.toString(); } + } else if (feature is PrimaryKey && column.type == ColumnType.integer) { + // this field is only relevant for integer columns because an INTEGER + // PRIMARY KEY is an alias for the rowid which should allow absent + // values during insert, even without the `AUTOINCREMENT` clause. + additionalParams['declaredAsPrimaryKey'] = 'true'; } } From 77e444b13b554c9637144de347859baa8688cfbe Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 23 Aug 2019 22:57:16 +0200 Subject: [PATCH 016/117] Verify that INTEGER PRIMARY KEY accepts absent values Verifies that #112 is fixed --- .../parsed_sql/moor_files_integration_test.dart | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/moor/test/parsed_sql/moor_files_integration_test.dart b/moor/test/parsed_sql/moor_files_integration_test.dart index d29a6953..4ea88f04 100644 --- a/moor/test/parsed_sql/moor_files_integration_test.dart +++ b/moor/test/parsed_sql/moor_files_integration_test.dart @@ -19,6 +19,10 @@ const _createConfig = 'CREATE TABLE IF NOT EXISTS config (' 'config_key VARCHAR not null primary key, ' 'config_value VARCHAR);'; +const _createMyTable = 'CREATE TABLE IF NOT EXISTS mytable (' + 'someid INTEGER NOT NULL PRIMARY KEY, ' + 'sometext VARCHAR);'; + void main() { // see ../data/tables/tables.moor test('creates tables as specified in .moor files', () async { @@ -31,6 +35,7 @@ void main() { verify(mockQueryExecutor.call(_createWithDefaults, [])); verify(mockQueryExecutor.call(_createWithConstraints, [])); verify(mockQueryExecutor.call(_createConfig, [])); + verify(mockQueryExecutor.call(_createMyTable, [])); }); test('infers primary keys correctly', () async { @@ -40,4 +45,13 @@ void main() { expect(db.withDefaults.primaryKey, isEmpty); expect(db.config.primaryKey, [db.config.configKey]); }); + + test('supports absent values for primary key integers', () async { + // regression test for #112: https://github.com/simolus3/moor/issues/112 + final mock = MockExecutor(); + final db = CustomTablesDb(mock); + + await db.into(db.mytable).insert(const MytableCompanion()); + verify(mock.runInsert('INSERT INTO mytable DEFAULT VALUES', [])); + }); } From 89f3987e6b0d6a35dde7077db1d8b212a3d3a2d7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 23 Aug 2019 23:07:31 +0200 Subject: [PATCH 017/117] Fix generation of PRIMARY KEY constraint on int column --- moor/lib/src/runtime/structure/columns.dart | 2 +- moor/test/columns/int_column_test.dart | 38 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 moor/test/columns/int_column_test.dart diff --git a/moor/lib/src/runtime/structure/columns.dart b/moor/lib/src/runtime/structure/columns.dart index 02055d7d..5d736638 100644 --- a/moor/lib/src/runtime/structure/columns.dart +++ b/moor/lib/src/runtime/structure/columns.dart @@ -206,7 +206,7 @@ class GeneratedIntColumn extends GeneratedColumn if (hasAutoIncrement) { into.write(' PRIMARY KEY AUTOINCREMENT'); } else if (declaredAsPrimaryKey) { - into.write('PRIMARY KEY'); + into.write(' PRIMARY KEY'); } } diff --git a/moor/test/columns/int_column_test.dart b/moor/test/columns/int_column_test.dart new file mode 100644 index 00000000..f9d4922a --- /dev/null +++ b/moor/test/columns/int_column_test.dart @@ -0,0 +1,38 @@ +import 'package:moor/src/runtime/components/component.dart'; +import 'package:test_api/test_api.dart'; +import 'package:moor/moor.dart'; + +import '../data/tables/todos.dart'; + +void main() { + test('int column writes AUTOINCREMENT constraint', () { + final column = GeneratedIntColumn( + 'foo', + 'tbl', + false, + declaredAsPrimaryKey: true, + hasAutoIncrement: true, + ); + + final context = GenerationContext.fromDb(TodoDb(null)); + column.writeColumnDefinition(context); + + expect( + context.sql, equals('foo INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT')); + }); + + test('int column writes PRIMARY KEY constraint', () { + final column = GeneratedIntColumn( + 'foo', + 'tbl', + false, + declaredAsPrimaryKey: true, + hasAutoIncrement: false, + ); + + final context = GenerationContext.fromDb(TodoDb(null)); + column.writeColumnDefinition(context); + + expect(context.sql, equals('foo INTEGER NOT NULL PRIMARY KEY')); + }); +} From ca76746000cfef53d2b33cee9f96956427b1aa7e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 24 Aug 2019 18:03:53 +0200 Subject: [PATCH 018/117] Better explain how `getSingle` and `watchSingle` works. --- moor/lib/src/runtime/statements/query.dart | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/moor/lib/src/runtime/statements/query.dart b/moor/lib/src/runtime/statements/query.dart index ce0018a0..7fc66e03 100644 --- a/moor/lib/src/runtime/statements/query.dart +++ b/moor/lib/src/runtime/statements/query.dart @@ -79,8 +79,21 @@ abstract class Selectable { /// result too many values, this method will throw. If no row is returned, /// `null` will be returned instead. /// + /// {@template moor_single_query_expl} /// Be aware that this operation won't put a limit clause on this statement, - /// if that's needed you would have to do that yourself. + /// if that's needed you would have to do use [SimpleSelectStatement.limit]: + /// ```dart + /// Future loadMostImportant() { + /// return (select(todos) + /// ..orderBy([(t) => OrderingTerm(expression: t.priority, mode: OrderingMode.desc)]) + /// ..limit(1) + /// ).getSingle(); + /// } + /// ``` + /// You should only use this method if you know the query won't have more than + /// one row, for instance because you used `limit(1)` or you know the `where` + /// clause will only allow one row. + /// {@endtemplate} Future getSingle() async { final list = await get(); final iterator = list.iterator; @@ -100,7 +113,10 @@ abstract class Selectable { /// However, it is assumed that the query will only emit one result, so /// instead of returning a [Stream>], this returns a [Stream]. If /// the query emits more than one row at some point, an error will be emitted - /// to the stream instead. + /// to the stream instead. If the query emits zero rows at some point, `null` + /// will be added to the stream instead. + /// + /// {@macro moor_single_query_expl} Stream watchSingle() { return watch().transform(singleElements()); } From 2b40272774c15fde2ea934b89bfed5c01e2ded3b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 24 Aug 2019 19:04:00 +0200 Subject: [PATCH 019/117] Explain how tables and columns are named in the docs --- docs/content/en/docs/Using SQL/custom_queries.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/content/en/docs/Using SQL/custom_queries.md b/docs/content/en/docs/Using SQL/custom_queries.md index 6ced9115..d918ffc5 100644 --- a/docs/content/en/docs/Using SQL/custom_queries.md +++ b/docs/content/en/docs/Using SQL/custom_queries.md @@ -39,6 +39,15 @@ Queries can have parameters in them by using the `?` or `:name` syntax. When you moor will figure out an appropriate type for them and include them in the generated methods. For instance, `'categoryById': 'SELECT * FROM categories WHERE id = :id'` will generate the method `categoryById(int id)`. +{{% alert title="On table names" color="info" %}} +To use this feature, it's helpful to know how Dart tables are named in sql. For tables that don't +override `tableName`, the name in sql will be the `snake_case` of the class name. So a Dart table +called `Categories` will be named `categories`, a table called `UserAddressInformation` would be +called `user_address_information`. The same rule applies to column getters without an explicit name. +Tables and columns declared in [SQL tables]({{< relref "custom_tables.md" >}}) will always have the +name you specified. +{{% /alert %}} + You can also use `UPDATE` or `DELETE` statements here. Of course, this feature is also available for [daos]({{< relref "../Advanced Features/daos.md" >}}), and it perfectly integrates with auto-updating streams by analyzing what tables you're reading from or From b1e6d931d5987e267c64efb5f96f95815b38c143 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 25 Aug 2019 22:44:28 +0200 Subject: [PATCH 020/117] Prepare release 1.7.1 --- moor/CHANGELOG.md | 5 +++++ moor/pubspec.yaml | 2 +- moor_generator/CHANGELOG.md | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 629372ff..4bdcc249 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.7.1 +- Better documentation on `getSingle` and `watchSingle` for queries. +- Fix `INTEGER NOT NULL PRIMARY KEY` wrongly requiring a value during insert (this never affected + `AUTOINCREMENT` columns, and only affects columns declared in a `.moor` file) + ## 1.7.0 - Support custom columns via type converters. See the [docs](https://moor.simonbinder.eu/type_converters) for details on how to use this feature. diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index 179b6f48..aaf3dd7c 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -1,6 +1,6 @@ name: moor description: Moor is a safe and reactive persistence library for Dart applications -version: 1.7.0 +version: 1.7.1 repository: https://github.com/simolus3/moor homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues diff --git a/moor_generator/CHANGELOG.md b/moor_generator/CHANGELOG.md index 3008ebb3..0a40e255 100644 --- a/moor_generator/CHANGELOG.md +++ b/moor_generator/CHANGELOG.md @@ -1,6 +1,7 @@ ## 1.7.1 - Drop support for analyzer versions `<0.36.4`. They weren't supported in version 1.7.0 either, but the `pubspec.yaml` did not specify this correctly. +- Support for moor version 1.7.1, which contains a fix for integer columns declared as primary key ## 1.7.0 - Support type converters that were introduced in moor 1.7 From 37672dad2da7faf26c7ecea91af82ac3118eedcc Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 26 Aug 2019 15:30:51 +0200 Subject: [PATCH 021/117] Bump moor dependency in moor_generator --- moor_generator/pubspec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moor_generator/pubspec.yaml b/moor_generator/pubspec.yaml index 84a87a13..97f2bbef 100644 --- a/moor_generator/pubspec.yaml +++ b/moor_generator/pubspec.yaml @@ -19,7 +19,7 @@ dependencies: source_span: ^1.5.5 build: ^1.1.0 build_config: '>=0.3.1 <1.0.0' - moor: ^1.7.0 + moor: ^1.7.1 meta: ^1.1.0 sqlparser: ^0.2.0 dev_dependencies: From fc6c9857bb68ad814154e405aa5b9ed1618d3ec5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 26 Aug 2019 21:11:26 +0200 Subject: [PATCH 022/117] Add example project to play around with the AS plugin --- extras/plugin_example/.gitignore | 35 +++++++++++++++++++++ extras/plugin_example/README.md | 24 ++++++++++++++ extras/plugin_example/analysis_options.yaml | 5 +++ extras/plugin_example/lib/test.dart | 1 + extras/plugin_example/lib/test.foobar | 3 ++ extras/plugin_example/lib/test.moor | 3 ++ extras/plugin_example/pubspec.yaml | 13 ++++++++ moor/tools/analyzer_plugin/pubspec.yaml | 10 +++--- 8 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 extras/plugin_example/.gitignore create mode 100644 extras/plugin_example/README.md create mode 100644 extras/plugin_example/analysis_options.yaml create mode 100644 extras/plugin_example/lib/test.dart create mode 100644 extras/plugin_example/lib/test.foobar create mode 100644 extras/plugin_example/lib/test.moor create mode 100644 extras/plugin_example/pubspec.yaml diff --git a/extras/plugin_example/.gitignore b/extras/plugin_example/.gitignore new file mode 100644 index 00000000..f02eadcc --- /dev/null +++ b/extras/plugin_example/.gitignore @@ -0,0 +1,35 @@ + +# Created by https://www.gitignore.io/api/dart,intellij +# Edit at https://www.gitignore.io/?templates=dart,intellij + +.vscode + +### Dart ### +# See https://www.dartlang.org/guides/libraries/private-files + +# Files and directories created by pub +.dart_tool/ +.packages +build/ +# If you're building an application, you may want to check-in your pubspec.lock +pubspec.lock + +# Directory created by dartdoc +# If you don't generate documentation locally you can remove this line. +doc/api/ + +# Avoid committing generated Javascript files: +*.dart.js +*.info.json # Produced by the --dump-info flag. +*.js # When generated by dart2js. Don't specify *.js if your + # project includes source files written in JavaScript. +*.js_ +*.js.deps +*.js.map + +android/ +ios/ + +### Intellij ### +.idea/**/* +# End of https://www.gitignore.io/api/dart,intellij diff --git a/extras/plugin_example/README.md b/extras/plugin_example/README.md new file mode 100644 index 00000000..d7d12cd4 --- /dev/null +++ b/extras/plugin_example/README.md @@ -0,0 +1,24 @@ +Playground to test the analyzer plugin for `.moor` files. + +## Playing around with this +At the moment, [DartCode](https://dartcode.org/) with version `v3.4.0-beta.2` is needed to run the +plugin. To set up the plugin, run the following steps + +1. Change the file `moor/tools/analyzer_plugin/pubspec.yaml` so that the `dependency_overrides` + section points to the location where you cloned this repository. This is needed because we + can't use relative paths for dependencies in analyzer plugins yet. +2. In VS Code, change `dart.additionalAnalyzerFileExtensions` to include `moor` files: + ```json + { + "dart.additionalAnalyzerFileExtensions": [ + "moor" + ] + } + ``` +3. If you already had the project open, close and re-open VS Code. Otherwise, simply open this + project. +4. Type around in a `.moor` file - notice how you still don't get syntax highlighting because + VS Code required a static grammar for files and can't use a language server for that :( + +Debugging plugins is not fun. See the [docs](https://github.com/dart-lang/sdk/blob/master/pkg/analyzer_plugin/doc/tutorial/debugging.md) +on some general guidance, and good luck. Enabling the analyzer diagnostics server can help. diff --git a/extras/plugin_example/analysis_options.yaml b/extras/plugin_example/analysis_options.yaml new file mode 100644 index 00000000..9ec18836 --- /dev/null +++ b/extras/plugin_example/analysis_options.yaml @@ -0,0 +1,5 @@ +include: package:pedantic/analysis_options.yaml + +analyzer: + plugins: + - moor \ No newline at end of file diff --git a/extras/plugin_example/lib/test.dart b/extras/plugin_example/lib/test.dart new file mode 100644 index 00000000..ef22a69f --- /dev/null +++ b/extras/plugin_example/lib/test.dart @@ -0,0 +1 @@ +class Test {} diff --git a/extras/plugin_example/lib/test.foobar b/extras/plugin_example/lib/test.foobar new file mode 100644 index 00000000..70ae6457 --- /dev/null +++ b/extras/plugin_example/lib/test.foobar @@ -0,0 +1,3 @@ +CREATE TABLE playground ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT +) \ No newline at end of file diff --git a/extras/plugin_example/lib/test.moor b/extras/plugin_example/lib/test.moor new file mode 100644 index 00000000..70ae6457 --- /dev/null +++ b/extras/plugin_example/lib/test.moor @@ -0,0 +1,3 @@ +CREATE TABLE playground ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT +) \ No newline at end of file diff --git a/extras/plugin_example/pubspec.yaml b/extras/plugin_example/pubspec.yaml new file mode 100644 index 00000000..7a8482b3 --- /dev/null +++ b/extras/plugin_example/pubspec.yaml @@ -0,0 +1,13 @@ +name: plugin_example +description: Playground to test the analyzer plugin + +environment: + sdk: '>=2.4.0 <3.0.0' + +dependencies: + moor: + path: ../../moor + +dev_dependencies: + pedantic: ^1.7.0 + test: ^1.5.0 diff --git a/moor/tools/analyzer_plugin/pubspec.yaml b/moor/tools/analyzer_plugin/pubspec.yaml index 194c4bbf..18ce1018 100644 --- a/moor/tools/analyzer_plugin/pubspec.yaml +++ b/moor/tools/analyzer_plugin/pubspec.yaml @@ -8,8 +8,8 @@ dependencies: # To work on this plugin, you need to add the absolute paths here. Relative paths aren't supported yet # https://github.com/dart-lang/sdk/issues/35281 -#dependency_overrides: -# moor_generator: -# path: /home/simon/IdeaProjects/moor/moor_generator -# sqlparser: -# path: /home/simon/IdeaProjects/moor/sqlparser +dependency_overrides: + moor_generator: + path: /home/simon/IdeaProjects/moor/moor_generator + sqlparser: + path: /home/simon/IdeaProjects/moor/sqlparser From 4b0add64de915cc8d9310f65f3755eb75c5e6b96 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 26 Aug 2019 22:26:38 +0200 Subject: [PATCH 023/117] Provide better error messages at unknown tables --- sqlparser/lib/src/analysis/analysis.dart | 1 + sqlparser/lib/src/analysis/error.dart | 37 +++++++++++++++++-- .../lib/src/analysis/schema/references.dart | 14 +++++-- .../src/analysis/steps/column_resolver.dart | 7 +++- .../lib/src/ast/expressions/reference.dart | 9 +++++ sqlparser/lib/src/reader/parser/crud.dart | 6 ++- 6 files changed, 63 insertions(+), 11 deletions(-) diff --git a/sqlparser/lib/src/analysis/analysis.dart b/sqlparser/lib/src/analysis/analysis.dart index d467477f..b52b5ea2 100644 --- a/sqlparser/lib/src/analysis/analysis.dart +++ b/sqlparser/lib/src/analysis/analysis.dart @@ -1,6 +1,7 @@ import 'dart:math'; import 'package:meta/meta.dart'; +import 'package:source_span/source_span.dart'; import 'package:sqlparser/sqlparser.dart'; import 'package:sqlparser/src/reader/tokenizer/token.dart'; diff --git a/sqlparser/lib/src/analysis/error.dart b/sqlparser/lib/src/analysis/error.dart index 6fad03d4..5dd805b1 100644 --- a/sqlparser/lib/src/analysis/error.dart +++ b/sqlparser/lib/src/analysis/error.dart @@ -7,20 +7,49 @@ class AnalysisError { AnalysisError({@required this.type, this.message, this.relevantNode}); - @override - String toString() { + /// The relevant portion of the source code that caused this error. Some AST + /// nodes don't have a span, in that case this error is going to be null. + SourceSpan get span { final first = relevantNode?.first?.span; final last = relevantNode?.last?.span; if (first != null && last != null) { - final span = first.expand(last); - return span.message(message ?? type.toString(), color: true); + return first.expand(last); + } + return null; + } + + @override + String toString() { + final msgSpan = span; + if (msgSpan != null) { + return msgSpan.message(message ?? type.toString(), color: true); } else { return 'Error: $type: $message at $relevantNode'; } } } +class UnresolvedReferenceError extends AnalysisError { + /// The attempted reference that couldn't be resolved + final String reference; + + /// A list of alternative references that would be available for [reference]. + final Iterable available; + + UnresolvedReferenceError( + {@required AnalysisErrorType type, + this.reference, + this.available, + AstNode relevantNode}) + : super(type: type, relevantNode: relevantNode); + + @override + String get message { + return 'Could not find $reference. Available are: ${available.join(', ')}'; + } +} + enum AnalysisErrorType { referencedUnknownTable, referencedUnknownColumn, diff --git a/sqlparser/lib/src/analysis/schema/references.dart b/sqlparser/lib/src/analysis/schema/references.dart index 5f8318d8..f6fbc6f4 100644 --- a/sqlparser/lib/src/analysis/schema/references.dart +++ b/sqlparser/lib/src/analysis/schema/references.dart @@ -13,7 +13,7 @@ mixin Referencable {} /// many things, basically only tables. /// /// For instance: "SELECT *, 1 AS d, (SELECT id FROM demo WHERE id = out.id) FROM demo AS out;" -/// is a valid sql query when the demo table as an id column. However, +/// is a valid sql query when the demo table has an id column. However, /// "SELECT *, 1 AS d, (SELECT id FROM demo WHERE id = d) FROM demo AS out;" is /// not, the "d" referencable is not visible for the child select statement. mixin VisibleToChildren on Referencable {} @@ -79,12 +79,20 @@ class ReferenceScope { /// Returns everything that is in scope and a subtype of [T]. List allOf() { var scope = this; + var isInCurrentScope = true; final collected = []; while (scope != null) { - collected.addAll( - scope._references.values.expand((list) => list).whereType()); + var foundValues = + scope._references.values.expand((list) => list).whereType(); + + if (!isInCurrentScope) { + foundValues = foundValues.whereType().cast(); + } + + collected.addAll(foundValues); scope = scope.parent; + isInCurrentScope = false; } return collected; } diff --git a/sqlparser/lib/src/analysis/steps/column_resolver.dart b/sqlparser/lib/src/analysis/steps/column_resolver.dart index 06029e2f..3d38d9c5 100644 --- a/sqlparser/lib/src/analysis/steps/column_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/column_resolver.dart @@ -115,10 +115,13 @@ class ColumnResolver extends RecursiveVisitor { Table _resolveTableReference(TableReference r) { final scope = r.scope; final resolvedTable = scope.resolve(r.tableName, orElse: () { - context.reportError(AnalysisError( + final available = scope.allOf
().map((t) => t.name); + + context.reportError(UnresolvedReferenceError( type: AnalysisErrorType.referencedUnknownTable, relevantNode: r, - message: 'The table ${r.tableName} could not be found', + reference: r.tableName, + available: available, )); }); return r.resolved = resolvedTable; diff --git a/sqlparser/lib/src/ast/expressions/reference.dart b/sqlparser/lib/src/ast/expressions/reference.dart index 4617016f..bba0a6d4 100644 --- a/sqlparser/lib/src/ast/expressions/reference.dart +++ b/sqlparser/lib/src/ast/expressions/reference.dart @@ -23,4 +23,13 @@ class Reference extends Expression with ReferenceOwner { bool contentEquals(Reference other) { return other.tableName == tableName && other.columnName == columnName; } + + @override + String toString() { + if (tableName != null) { + return '$tableName.$columnName'; + } else { + return columnName; + } + } } diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 951ce921..9545eb84 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -130,9 +130,11 @@ mixin CrudParser on ParserBase { if (_matchOne(TokenType.identifier)) { // ignore the schema name, it's not supported. Besides that, we're on the // first branch in the diagram here - final tableName = (_previous as IdentifierToken).identifier; + final firstToken = _previous as IdentifierToken; + final tableName = firstToken.identifier; final alias = _as(); - return TableReference(tableName, alias?.identifier); + return TableReference(tableName, alias?.identifier) + ..setSpan(firstToken, _previous); } return null; } From 3612c78241311aec24650045827c5a3d98789194 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 27 Aug 2019 11:54:56 +0200 Subject: [PATCH 024/117] Introduce special token class for keywords --- sqlparser/lib/src/reader/tokenizer/scanner.dart | 2 +- sqlparser/lib/src/reader/tokenizer/token.dart | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sqlparser/lib/src/reader/tokenizer/scanner.dart b/sqlparser/lib/src/reader/tokenizer/scanner.dart index 989d56bd..1dc48072 100644 --- a/sqlparser/lib/src/reader/tokenizer/scanner.dart +++ b/sqlparser/lib/src/reader/tokenizer/scanner.dart @@ -306,7 +306,7 @@ class Scanner { // not escaped, so it could be a keyword final text = _currentSpan.text.toUpperCase(); if (keywords.containsKey(text)) { - _addToken(keywords[text]); + tokens.add(KeywordToken(keywords[text], _currentSpan)); } else { tokens.add(IdentifierToken(false, _currentSpan)); } diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index e7e9f500..368f6adb 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -266,6 +266,13 @@ class IdentifierToken extends Token { : super(TokenType.identifier, span); } +/// Used for tokens that are keywords. We use this special class without any +/// additional properties to ease syntax highlighting, as it allows us to find +/// the keywords easily. +class KeywordToken extends Token { + KeywordToken(TokenType type, FileSpan span) : super(type, span); +} + class TokenizerError { final String message; final SourceLocation location; From aa13aad27699452c8475ce2b28c005f20e9b1623 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 27 Aug 2019 12:15:32 +0200 Subject: [PATCH 025/117] Introduce moor-specific tokens for type converters --- sqlparser/lib/src/engine/sql_engine.dart | 9 ++++- .../lib/src/reader/tokenizer/scanner.dart | 38 ++++++++++++++++++- sqlparser/lib/src/reader/tokenizer/token.dart | 19 ++++++++++ sqlparser/test/scanner/moor_tokens_test.dart | 26 +++++++++++++ 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 sqlparser/test/scanner/moor_tokens_test.dart diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index bdda0f2c..35d46c49 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -8,7 +8,12 @@ class SqlEngine { /// All tables registered with [registerTable]. final List
knownTables = []; - SqlEngine(); + /// Moor extends the sql grammar a bit to support type converters and other + /// features. Enabling this flag will make this engine parse sql with these + /// extensions enabled. + final bool useMoorExtensions; + + SqlEngine({this.useMoorExtensions = false}); /// Registers the [table], which means that it can later be used in sql /// statements. @@ -28,7 +33,7 @@ class SqlEngine { /// Tokenizes the [source] into a list list [Token]s. Each [Token] contains /// information about where it appears in the [source] and a [TokenType]. List tokenize(String source) { - final scanner = Scanner(source); + final scanner = Scanner(source, scanMoorTokens: useMoorExtensions); final tokens = scanner.scanTokens(); if (scanner.errors.isNotEmpty) { diff --git a/sqlparser/lib/src/reader/tokenizer/scanner.dart b/sqlparser/lib/src/reader/tokenizer/scanner.dart index 1dc48072..d388f572 100644 --- a/sqlparser/lib/src/reader/tokenizer/scanner.dart +++ b/sqlparser/lib/src/reader/tokenizer/scanner.dart @@ -4,6 +4,9 @@ import 'package:sqlparser/src/reader/tokenizer/utils.dart'; class Scanner { final String source; + + /// Whether to scan tokens that are only relevant for moor. + final bool scanMoorTokens; final SourceFile _file; final List tokens = []; @@ -21,7 +24,8 @@ class Scanner { return _file.location(_currentOffset); } - Scanner(this.source) : _file = SourceFile.fromString(source); + Scanner(this.source, {this.scanMoorTokens = false}) + : _file = SourceFile.fromString(source); List scanTokens() { while (!_isAtEnd) { @@ -131,6 +135,13 @@ class Scanner { // todo sqlite also allows string literals with double ticks, we don't _identifier(escapedInQuotes: true); break; + case '`': + if (scanMoorTokens) { + _inlineDart(); + } else { + _unexpectedToken(); + } + break; case ' ': case '\t': case '\n': @@ -143,12 +154,16 @@ class Scanner { } else if (canStartColumnName(char)) { _identifier(); } else { - errors.add(TokenizerError('Unexpected character.', _currentLocation)); + _unexpectedToken(); } break; } } + void _unexpectedToken() { + errors.add(TokenizerError('Unexpected character.', _currentLocation)); + } + String _nextChar() { _currentOffset++; return source.substring(_currentOffset - 1, _currentOffset); @@ -307,9 +322,28 @@ class Scanner { final text = _currentSpan.text.toUpperCase(); if (keywords.containsKey(text)) { tokens.add(KeywordToken(keywords[text], _currentSpan)); + } else if (scanMoorTokens && moorKeywords.containsKey(text)) { + tokens.add(KeywordToken(moorKeywords[text], _currentSpan)); } else { tokens.add(IdentifierToken(false, _currentSpan)); } } } + + void _inlineDart() { + // inline starts with a `, we just need to find the matching ` that + // terminates this token. + while (_peek() != '`' && !_isAtEnd) { + _nextChar(); + } + + if (_isAtEnd) { + errors.add( + TokenizerError('Unterminated inline Dart code', _currentLocation)); + } else { + // consume the ` + _nextChar(); + tokens.add(InlineDartToken(_currentSpan)); + } + } } diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index 368f6adb..b7c7251d 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -133,6 +133,10 @@ enum TokenType { semicolon, eof, + + /// Moor specific token, used to declare a type converters + mapped, + inlineDart, } const Map keywords = { @@ -226,6 +230,10 @@ const Map keywords = { 'WINDOW': TokenType.window, }; +const Map moorKeywords = { + 'MAPPED': TokenType.mapped, +}; + class Token { final TokenType type; @@ -266,6 +274,17 @@ class IdentifierToken extends Token { : super(TokenType.identifier, span); } +/// Inline Dart appearing in a create table statement. Only parsed when the moor +/// extensions are enabled. Dart code is wrapped in backticks. +class InlineDartToken extends Token { + InlineDartToken(FileSpan span) : super(TokenType.inlineDart, span); + + String get dartCode { + // strip the backticks + return lexeme.substring(1, lexeme.length - 1); + } +} + /// Used for tokens that are keywords. We use this special class without any /// additional properties to ease syntax highlighting, as it allows us to find /// the keywords easily. diff --git a/sqlparser/test/scanner/moor_tokens_test.dart b/sqlparser/test/scanner/moor_tokens_test.dart new file mode 100644 index 00000000..0c863bd2 --- /dev/null +++ b/sqlparser/test/scanner/moor_tokens_test.dart @@ -0,0 +1,26 @@ +import 'package:test/test.dart'; +import 'package:sqlparser/src/reader/tokenizer/scanner.dart'; +import 'package:sqlparser/src/reader/tokenizer/token.dart'; + +void main() { + test('parses moor specific tokens', () { + final part = 'c INTEGER MAPPED BY `const Mapper()` NOT NULL'; + final scanner = Scanner(part, scanMoorTokens: true); + final tokens = scanner.scanTokens(); + + expect(scanner.errors, isEmpty); + expect(tokens.map((t) => t.type), [ + TokenType.identifier, // c + TokenType.identifier, // INTEGER + TokenType.mapped, + TokenType.by, + TokenType.inlineDart, // `const Mapper()` + TokenType.not, + TokenType.$null, + TokenType.eof, + ]); + + expect( + tokens.whereType().single.dartCode, 'const Mapper()'); + }); +} From 5d2149d72768c9af1af1c46238b46e2692b05b87 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 27 Aug 2019 12:33:48 +0200 Subject: [PATCH 026/117] Parse MAPPED BY constraints for moor files --- .../lib/src/ast/schema/column_definition.dart | 20 ++++++++++++++++++ sqlparser/lib/src/engine/sql_engine.dart | 2 +- sqlparser/lib/src/reader/parser/parser.dart | 8 +++++-- sqlparser/lib/src/reader/parser/schema.dart | 9 ++++++++ sqlparser/test/parser/create_table_test.dart | 21 +++++++++++++++++++ sqlparser/test/parser/utils.dart | 5 +++++ 6 files changed, 62 insertions(+), 3 deletions(-) diff --git a/sqlparser/lib/src/ast/schema/column_definition.dart b/sqlparser/lib/src/ast/schema/column_definition.dart index d696ecd8..d875e5fb 100644 --- a/sqlparser/lib/src/ast/schema/column_definition.dart +++ b/sqlparser/lib/src/ast/schema/column_definition.dart @@ -40,6 +40,7 @@ abstract class ColumnConstraint extends AstNode { T Function(Default) isDefault, T Function(CollateConstraint) collate, T Function(ForeignKeyColumnConstraint) foreignKey, + T Function(MappedBy) mappedBy, }) { if (this is NotNull) { return notNull?.call(this as NotNull); @@ -55,6 +56,8 @@ abstract class ColumnConstraint extends AstNode { return collate?.call(this as CollateConstraint); } else if (this is ForeignKeyColumnConstraint) { return foreignKey?.call(this as ForeignKeyColumnConstraint); + } else if (this is MappedBy) { + return mappedBy?.call(this as MappedBy); } else { throw Exception('Did not expect $runtimeType as a ColumnConstraint'); } @@ -164,3 +167,20 @@ class ForeignKeyColumnConstraint extends ColumnConstraint { @override Iterable get childNodes => [clause]; } + +/// A `MAPPED BY` constraint, which is only parsed for moor files. It can be +/// used to declare a type converter for this column. +class MappedBy extends ColumnConstraint { + /// The Dart expression creating the type converter we use to map this token. + final InlineDartToken mapper; + + MappedBy(String name, this.mapper) : super(name); + + @override + bool _equalToConstraint(MappedBy other) { + return other.mapper.dartCode == mapper.dartCode; + } + + @override + final Iterable childNodes = const []; +} diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 35d46c49..f8061a38 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -46,7 +46,7 @@ class SqlEngine { /// Parses the [sql] statement into an AST-representation. ParseResult parse(String sql) { final tokens = tokenize(sql); - final parser = Parser(tokens); + final parser = Parser(tokens, useMoor: useMoorExtensions); final stmt = parser.statement(); return ParseResult._(stmt, parser.errors, sql); diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 66d0fecf..8fc1de35 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -43,9 +43,13 @@ class ParsingError implements Exception { abstract class ParserBase { final List tokens; final List errors = []; + + /// Whether to enable the extensions moor makes to the sql grammar. + final bool enableMoorExtensions; + int _current = 0; - ParserBase(this.tokens); + ParserBase(this.tokens, this.enableMoorExtensions); bool get _isAtEnd => _peek.type == TokenType.eof; Token get _peek => tokens[_current]; @@ -145,7 +149,7 @@ abstract class ParserBase { class Parser extends ParserBase with ExpressionParser, SchemaParser, CrudParser { - Parser(List tokens) : super(tokens); + Parser(List tokens, {bool useMoor = false}) : super(tokens, useMoor); Statement statement({bool expectEnd = true}) { final first = _peek; diff --git a/sqlparser/lib/src/reader/parser/schema.dart b/sqlparser/lib/src/reader/parser/schema.dart index 7b429ff0..04634318 100644 --- a/sqlparser/lib/src/reader/parser/schema.dart +++ b/sqlparser/lib/src/reader/parser/schema.dart @@ -153,6 +153,15 @@ mixin SchemaParser on ParserBase { return ForeignKeyColumnConstraint(resolvedName, clause) ..setSpan(first, _previous); } + if (enableMoorExtensions && _matchOne(TokenType.mapped)) { + _consume(TokenType.by, 'Expected a MAPPED BY constraint'); + + final dartExpr = _consume( + TokenType.inlineDart, 'Expected Dart expression in backticks'); + + return MappedBy(resolvedName, dartExpr as InlineDartToken) + ..setSpan(first, _previous); + } // no known column constraint matched. If orNull is set and we're not // guaranteed to be in a constraint clause (started with CONSTRAINT), we diff --git a/sqlparser/test/parser/create_table_test.dart b/sqlparser/test/parser/create_table_test.dart index c80fbad2..7a5c2fe3 100644 --- a/sqlparser/test/parser/create_table_test.dart +++ b/sqlparser/test/parser/create_table_test.dart @@ -1,5 +1,6 @@ import 'package:sqlparser/sqlparser.dart'; import 'package:sqlparser/src/ast/ast.dart'; +import 'package:sqlparser/src/utils/ast_equality.dart'; import 'package:test_core/test_core.dart'; import '../common_data.dart'; @@ -115,4 +116,24 @@ void main() { ), ); }); + + test('parses MAPPED BY expressions when in moor mode', () { + const stmt = 'CREATE TABLE a (b NOT NULL MAPPED BY `Mapper()` PRIMARY KEY)'; + final parsed = SqlEngine(useMoorExtensions: true).parse(stmt).rootNode; + + enforceEqual( + parsed, + CreateTableStatement(tableName: 'a', columns: [ + ColumnDefinition( + columnName: 'b', + typeName: null, + constraints: [ + NotNull(null), + MappedBy(null, inlineDart('Mapper()')), + PrimaryKeyColumn(null), + ], + ), + ]), + ); + }); } diff --git a/sqlparser/test/parser/utils.dart b/sqlparser/test/parser/utils.dart index 93df4b3c..e7b3b75f 100644 --- a/sqlparser/test/parser/utils.dart +++ b/sqlparser/test/parser/utils.dart @@ -10,6 +10,11 @@ Token token(TokenType type) { return Token(type, null); } +InlineDartToken inlineDart(String dartCode) { + final fakeFile = SourceFile.fromString('`$dartCode`'); + return InlineDartToken(fakeFile.span(0)); +} + IdentifierToken identifier(String content) { final fakeFile = SourceFile.fromString(content); return IdentifierToken(false, fakeFile.span(0)); From 0860b6645abfc41e87575de7c710aa335d37f7f4 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 27 Aug 2019 18:44:34 +0200 Subject: [PATCH 027/117] Generate selectable for custom queries This makes is easier to use getSingle() and watchSingle() on them. Fixes #120 --- .../tests/lib/suite/transactions.dart | 2 + moor/example/example.g.dart | 21 +-- moor/lib/src/runtime/database.dart | 41 +++++- moor/lib/src/runtime/statements/query.dart | 27 ++++ moor/test/data/tables/todos.g.dart | 129 ++++++++++-------- .../example/lib/database/database.dart | 4 +- .../lib/src/parser/moor/parsed_moor_file.dart | 16 ++- .../lib/src/writer/query_writer.dart | 82 +++++++---- 8 files changed, 216 insertions(+), 106 deletions(-) diff --git a/extras/integration_tests/tests/lib/suite/transactions.dart b/extras/integration_tests/tests/lib/suite/transactions.dart index 93b55eef..9b0e41ad 100644 --- a/extras/integration_tests/tests/lib/suite/transactions.dart +++ b/extras/integration_tests/tests/lib/suite/transactions.dart @@ -8,6 +8,7 @@ void transactionTests(TestExecutor executor) { test('transactions write data', () async { final db = Database(executor.createExecutor()); + // ignore: invalid_use_of_protected_member, invalid_use_of_visible_for_testing_member await db.transaction((_) async { final florianId = await db.writeUser(People.florian); @@ -30,6 +31,7 @@ void transactionTests(TestExecutor executor) { final db = Database(executor.createExecutor()); try { + // ignore: invalid_use_of_protected_member, invalid_use_of_visible_for_testing_member await db.transaction((_) async { final florianId = await db.writeUser(People.florian); diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index cdf8da0b..31184953 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -832,22 +832,23 @@ abstract class _$Database extends GeneratedDatabase { ); } + Selectable _totalWeightQuery( + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + ' SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id\n ', + variables: [], + readsFrom: {recipes, ingredientInRecipes}).map(_rowToTotalWeightResult); + } + Future> _totalWeight( {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - ' SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id\n ', - variables: []).then((rows) => rows.map(_rowToTotalWeightResult).toList()); + return _totalWeightQuery(operateOn: operateOn).get(); } Stream> _watchTotalWeight() { - return customSelectStream( - ' SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id\n ', - variables: [], - readsFrom: { - recipes, - ingredientInRecipes - }).map((rows) => rows.map(_rowToTotalWeightResult).toList()); + return _totalWeightQuery().watch(); } @override diff --git a/moor/lib/src/runtime/database.dart b/moor/lib/src/runtime/database.dart index 61e472a8..034cc53d 100644 --- a/moor/lib/src/runtime/database.dart +++ b/moor/lib/src/runtime/database.dart @@ -111,6 +111,7 @@ mixin QueryEngine on DatabaseConnectionUser { /// although it is very likely that the user meant to call it on the /// [Transaction] t. We can detect this by calling the function passed to /// `transaction` in a forked [Zone] storing the transaction in + @protected bool get topLevel => false; /// We can detect when a user called methods on the wrong [QueryEngine] @@ -169,6 +170,8 @@ mixin QueryEngine on DatabaseConnectionUser { /// You can use the [updates] parameter so that moor knows which tables are /// affected by your query. All select streams that depend on a table /// specified there will then issue another query. + @protected + @visibleForTesting Future customUpdate(String query, {List variables = const [], Set updates}) async { final engine = _resolvedEngine; @@ -190,11 +193,12 @@ mixin QueryEngine on DatabaseConnectionUser { /// Executes a custom select statement once. To use the variables, mark them /// with a "?" in your [query]. They will then be changed to the appropriate /// value. + @protected + @visibleForTesting + @Deprecated('use customSelectQuery(...).get() instead') Future> customSelect(String query, {List variables = const []}) async { - return CustomSelectStatement( - query, variables, {}, _resolvedEngine) - .get(); + return customSelectQuery(query, variables: variables).get(); } /// Creates a stream from a custom select statement.To use the variables, mark @@ -202,15 +206,36 @@ mixin QueryEngine on DatabaseConnectionUser { /// appropriate value. The stream will re-emit items when any table in /// [readsFrom] changes, so be sure to set it to the set of tables your query /// reads data from. + @protected + @visibleForTesting + @Deprecated('use customSelectQuery(...).watch() instead') Stream> customSelectStream(String query, {List variables = const [], Set readsFrom}) { - final tables = readsFrom ?? {}; - final statement = - CustomSelectStatement(query, variables, tables, _resolvedEngine); - return statement.watch(); + return customSelectQuery(query, variables: variables, readsFrom: readsFrom) + .watch(); + } + + /// Creates a custom select statement from the given sql [query]. To run the + /// query once, use [Selectable.get]. For an auto-updating streams, set the + /// set of tables the ready [readsFrom] and use [Selectable.watch]. If you + /// know the query will never emit more than one row, you can also use + /// [Selectable.getSingle] and [Selectable.watchSingle] which return the item + /// directly or wrapping it into a list. + /// + /// If you use variables in your query (for instance with "?"), they will be + /// bound to the [variables] you specify on this query. + @protected + @visibleForTesting + Selectable customSelectQuery(String query, + {List variables = const [], + Set readsFrom = const {}}) { + readsFrom ??= {}; + return CustomSelectStatement(query, variables, readsFrom, _resolvedEngine); } /// Executes the custom sql [statement] on the database. + @protected + @visibleForTesting Future customStatement(String statement) { return _resolvedEngine.executor.runCustom(statement); } @@ -226,6 +251,8 @@ mixin QueryEngine on DatabaseConnectionUser { /// might be different than that of the "global" database instance. /// 2. Nested transactions are not supported. Creating another transaction /// inside a transaction returns the parent transaction. + @protected + @visibleForTesting Future transaction(Future Function(QueryEngine transaction) action) async { final resolved = _resolvedEngine; if (resolved is Transaction) { diff --git a/moor/lib/src/runtime/statements/query.dart b/moor/lib/src/runtime/statements/query.dart index 7fc66e03..80bbf8e1 100644 --- a/moor/lib/src/runtime/statements/query.dart +++ b/moor/lib/src/runtime/statements/query.dart @@ -120,6 +120,33 @@ abstract class Selectable { Stream watchSingle() { return watch().transform(singleElements()); } + + /// Maps this selectable by using [mapper]. + /// + /// Each entry emitted by this [Selectable] will be transformed by the + /// [mapper] and then emitted to the selectable returned. + Selectable map(N Function(T) mapper) { + return _MappedSelectable(this, mapper); + } +} + +class _MappedSelectable extends Selectable { + final Selectable _source; + final T Function(S) _mapper; + + _MappedSelectable(this._source, this._mapper); + + @override + Future> get() { + return _source.get().then(_mapResults); + } + + @override + Stream> watch() { + return _source.watch().map(_mapResults); + } + + List _mapResults(List results) => results.map(_mapper).toList(); } mixin SingleTableQueryMixin diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 0caf9e30..5d5bbce7 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1303,22 +1303,26 @@ abstract class _$TodoDb extends GeneratedDatabase { ); } - Future> allTodosWithCategory( + Selectable allTodosWithCategoryQuery( {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT t.*, c.id as catId, c."desc" as catDesc FROM todos t INNER JOIN categories c ON c.id = t.category', - variables: []).then((rows) => rows.map(_rowToAllTodosWithCategoryResult).toList()); - } - - Stream> watchAllTodosWithCategory() { - return customSelectStream( + return (operateOn ?? this).customSelectQuery( 'SELECT t.*, c.id as catId, c."desc" as catDesc FROM todos t INNER JOIN categories c ON c.id = t.category', variables: [], readsFrom: { categories, todosTable - }).map((rows) => rows.map(_rowToAllTodosWithCategoryResult).toList()); + }).map(_rowToAllTodosWithCategoryResult); + } + + Future> allTodosWithCategory( + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return allTodosWithCategoryQuery(operateOn: operateOn).get(); + } + + Stream> watchAllTodosWithCategory() { + return allTodosWithCategoryQuery().watch(); } Future deleteTodoById( @@ -1344,7 +1348,7 @@ abstract class _$TodoDb extends GeneratedDatabase { ); } - Future> withIn( + Selectable withInQuery( String var1, String var2, List var3, @@ -1353,21 +1357,7 @@ abstract class _$TodoDb extends GeneratedDatabase { var $highestIndex = 3; final expandedvar3 = $expandVar($highestIndex, var3.length); $highestIndex += var3.length; - return (operateOn ?? this).customSelect( - 'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1', - variables: [ - Variable.withString(var1), - Variable.withString(var2), - for (var $ in var3) Variable.withInt($), - ]).then((rows) => rows.map(_rowToTodoEntry).toList()); - } - - Stream> watchWithIn( - String var1, String var2, List var3) { - var $highestIndex = 3; - final expandedvar3 = $expandVar($highestIndex, var3.length); - $highestIndex += var3.length; - return customSelectStream( + return (operateOn ?? this).customSelectQuery( 'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1', variables: [ Variable.withString(var1), @@ -1376,29 +1366,46 @@ abstract class _$TodoDb extends GeneratedDatabase { ], readsFrom: { todosTable - }).map((rows) => rows.map(_rowToTodoEntry).toList()); + }).map(_rowToTodoEntry); + } + + Future> withIn( + String var1, + String var2, + List var3, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return withInQuery(var1, var2, var3, operateOn: operateOn).get(); + } + + Stream> watchWithIn( + String var1, String var2, List var3) { + return withInQuery(var1, var2, var3).watch(); + } + + Selectable searchQuery( + int id, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'SELECT * FROM todos WHERE CASE WHEN -1 = :id THEN 1 ELSE id = :id END', + variables: [ + Variable.withInt(id), + ], + readsFrom: { + todosTable + }).map(_rowToTodoEntry); } Future> search( int id, {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT * FROM todos WHERE CASE WHEN -1 = :id THEN 1 ELSE id = :id END', - variables: [ - Variable.withInt(id), - ]).then((rows) => rows.map(_rowToTodoEntry).toList()); + return searchQuery(id, operateOn: operateOn).get(); } Stream> watchSearch(int id) { - return customSelectStream( - 'SELECT * FROM todos WHERE CASE WHEN -1 = :id THEN 1 ELSE id = :id END', - variables: [ - Variable.withInt(id), - ], - readsFrom: { - todosTable - }).map((rows) => rows.map(_rowToTodoEntry).toList()); + return searchQuery(id).watch(); } FindCustomResult _rowToFindCustomResult(QueryRow row) { @@ -1408,20 +1415,23 @@ abstract class _$TodoDb extends GeneratedDatabase { ); } + Selectable findCustomQuery( + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'SELECT custom FROM table_without_p_k WHERE some_float < 10', + variables: [], + readsFrom: {tableWithoutPK}).map(_rowToFindCustomResult); + } + Future> findCustom( {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT custom FROM table_without_p_k WHERE some_float < 10', - variables: []).then((rows) => rows.map(_rowToFindCustomResult).toList()); + return findCustomQuery(operateOn: operateOn).get(); } Stream> watchFindCustom() { - return customSelectStream( - 'SELECT custom FROM table_without_p_k WHERE some_float < 10', - variables: [], - readsFrom: {tableWithoutPK}) - .map((rows) => rows.map(_rowToFindCustomResult).toList()); + return findCustomQuery().watch(); } @override @@ -1453,19 +1463,11 @@ mixin _$SomeDaoMixin on DatabaseAccessor { ); } - Future> todosForUser( + Selectable todosForUserQuery( int user, {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', - variables: [ - Variable.withInt(user), - ]).then((rows) => rows.map(_rowToTodoEntry).toList()); - } - - Stream> watchTodosForUser(int user) { - return customSelectStream( + return (operateOn ?? this).customSelectQuery( 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', variables: [ Variable.withInt(user), @@ -1474,6 +1476,17 @@ mixin _$SomeDaoMixin on DatabaseAccessor { todosTable, sharedTodos, users - }).map((rows) => rows.map(_rowToTodoEntry).toList()); + }).map(_rowToTodoEntry); + } + + Future> todosForUser( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return todosForUserQuery(user, operateOn: operateOn).get(); + } + + Stream> watchTodosForUser(int user) { + return todosForUserQuery(user).watch(); } } diff --git a/moor_flutter/example/lib/database/database.dart b/moor_flutter/example/lib/database/database.dart index d0639556..e48b539d 100644 --- a/moor_flutter/example/lib/database/database.dart +++ b/moor_flutter/example/lib/database/database.dart @@ -91,14 +91,14 @@ class Database extends _$Database { Stream> categoriesWithCount() { // select all categories and load how many associated entries there are for // each category - return customSelectStream( + return customSelectQuery( 'SELECT c.id, c.desc, ' '(SELECT COUNT(*) FROM todos WHERE category = c.id) AS amount ' 'FROM categories c ' 'UNION ALL SELECT null, null, ' '(SELECT COUNT(*) FROM todos WHERE category IS NULL)', readsFrom: {todos, categories}, - ).map((rows) { + ).watch().map((rows) { // when we have the result set, map each row to the data class return rows.map((row) { final hasId = row.data['id'] != null; diff --git a/moor_generator/lib/src/parser/moor/parsed_moor_file.dart b/moor_generator/lib/src/parser/moor/parsed_moor_file.dart index b9f712cf..aabdae85 100644 --- a/moor_generator/lib/src/parser/moor/parsed_moor_file.dart +++ b/moor_generator/lib/src/parser/moor/parsed_moor_file.dart @@ -1,5 +1,6 @@ import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/used_type_converter.dart'; import 'package:moor_generator/src/parser/sql/type_mapping.dart'; import 'package:moor_generator/src/utils/names.dart'; import 'package:moor_generator/src/utils/string_escaper.dart'; @@ -33,6 +34,8 @@ class CreateTable { /// The AST of this `CREATE TABLE` statement. final ParseResult ast; + CreateTable(this.ast); + SpecifiedTable extractTable(TypeMapper mapper) { final table = SchemaFromCreateTable().read(ast.rootNode as CreateTableStatement); @@ -47,6 +50,7 @@ class CreateTable { final dartName = ReCase(sqlName).camelCase; final constraintWriter = StringBuffer(); final moorType = mapper.resolvedToMoor(column.type); + UsedTypeConverter converter; String defaultValue; for (var constraint in column.constraints) { @@ -65,6 +69,12 @@ class CreateTable { defaultValue = '$expressionName(${asDartLiteral(sqlDefault)})'; } + if (constraint is MappedBy) { + converter = _readTypeConverter(constraint); + // don't write MAPPED BY constraints when creating the table + continue; + } + if (constraintWriter.isNotEmpty) { constraintWriter.write(' '); } @@ -79,6 +89,7 @@ class CreateTable { features: features, customConstraints: constraintWriter.toString(), defaultArgument: defaultValue, + typeConverter: converter, ); foundColumns[column.name] = parsed; @@ -114,5 +125,8 @@ class CreateTable { ); } - CreateTable(this.ast); + UsedTypeConverter _readTypeConverter(MappedBy mapper) { + // todo we need to somehow parse the dart expression and check types + return null; + } } diff --git a/moor_generator/lib/src/writer/query_writer.dart b/moor_generator/lib/src/writer/query_writer.dart index 7c45fe4e..8c98d325 100644 --- a/moor_generator/lib/src/writer/query_writer.dart +++ b/moor_generator/lib/src/writer/query_writer.dart @@ -46,6 +46,7 @@ class QueryWriter { void _writeSelect(StringBuffer buffer) { _writeMapping(buffer); + _writeSelectStatementCreator(buffer); _writeOneTimeReader(buffer); _writeStreamReader(buffer); } @@ -54,6 +55,10 @@ class QueryWriter { return '_rowTo${_select.resultClassName}'; } + String _nameOfCreationMethod() { + return '${_select.name}Query'; + } + /// Writes a mapping method that turns a "QueryRow" into the desired custom /// return type. void _writeMapping(StringBuffer buffer) { @@ -87,27 +92,50 @@ class QueryWriter { } } + /// Writes a method returning a `Selectable`, where `T` is the return type + /// of the custom query. + void _writeSelectStatementCreator(StringBuffer buffer) { + final returnType = 'Selectable<${_select.resultClassName}>'; + final methodName = _nameOfCreationMethod(); + + buffer.write('$returnType $methodName('); + _writeParameters(buffer); + buffer.write(') {\n'); + + _writeExpandedDeclarations(buffer); + buffer + ..write('return (operateOn ?? this).') + ..write('customSelectQuery(${_queryCode()}, '); + _writeVariables(buffer); + buffer.write(', '); + _writeReadsFrom(buffer); + + buffer.write(').map('); + buffer.write(_nameOfMappingMethod()); + buffer.write(');\n}\n'); + } + + /* + Future> allTodos(String name, + {QueryEngine overrideEngine}) { + return _allTodosWithCategoryQuery(name, engine: overrideEngine).get(); + } + */ + void _writeOneTimeReader(StringBuffer buffer) { buffer.write('Future> ${query.name}('); _writeParameters(buffer); - buffer.write(') {\n'); - _writeExpandedDeclarations(buffer); - buffer - ..write('return (operateOn ?? this).') // use custom engine, if set - ..write('customSelect(${_queryCode()},'); - _writeVariables(buffer); - buffer - ..write(')') - ..write( - '.then((rows) => rows.map(${_nameOfMappingMethod()}).toList());\n') - ..write('\n}\n'); + buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); + _writeUseParameters(buffer); + buffer.write(').get();\n}\n'); } void _writeStreamReader(StringBuffer buffer) { - // turning the query name into pascal case will remove underscores final upperQueryName = ReCase(query.name).pascalCase; String methodName; + // turning the query name into pascal case will remove underscores, add the + // "private" modifier back in if needed if (session.options.fixPrivateWatchMethods && query.name.startsWith('_')) { methodName = '_watch$upperQueryName'; } else { @@ -115,23 +143,10 @@ class QueryWriter { } buffer.write('Stream> $methodName('); - // don't supply an engine override parameter because select streams cannot - // be used in transaction or similar context, only on the main database - // engine. _writeParameters(buffer, dontOverrideEngine: true); - buffer.write(') {\n'); - - _writeExpandedDeclarations(buffer); - buffer..write('return customSelectStream(${_queryCode()},'); - - _writeVariables(buffer); - buffer.write(','); - _writeReadsFrom(buffer); - - buffer - ..write(')') - ..write('.map((rows) => rows.map(${_nameOfMappingMethod()}).toList());\n') - ..write('\n}\n'); + buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); + _writeUseParameters(buffer, dontUseEngine: true); + buffer.write(').watch();\n}\n'); } void _writeUpdatingQuery(StringBuffer buffer) { @@ -177,6 +192,17 @@ class QueryWriter { } } + /// Writes code that uses the parameters as declared by [_writeParameters], + /// assuming that for each parameter, a variable with the same name exists + /// in the current scope. + void _writeUseParameters(StringBuffer into, {bool dontUseEngine = false}) { + into.write(query.variables.map((v) => v.dartParameterName).join(', ')); + if (!dontUseEngine) { + if (query.variables.isNotEmpty) into.write(', '); + into.write('operateOn: operateOn'); + } + } + // Some notes on parameters and generating query code: // We expand array parameters to multiple variables at runtime (see the // documentation of FoundVariable and SqlQuery for further discussion). From 03a35ae9cd454c8c6380bfaa97e3ef7fed08db9a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 28 Aug 2019 16:57:25 +0200 Subject: [PATCH 028/117] Port ffi example to updated structs / pointer api --- moor/lib/src/vm/api/database.dart | 35 ++++---- moor/lib/src/vm/api/errors.dart | 12 ++- moor/lib/src/vm/api/prepared_statement.dart | 24 +++--- moor/lib/src/vm/bindings/bindings.dart | 78 ++++++++++-------- moor/lib/src/vm/bindings/signatures.dart | 88 ++++++++++++--------- moor/lib/src/vm/bindings/types.dart | 8 +- moor/lib/src/vm/ffi/blob.dart | 75 +++++++++--------- moor/lib/src/vm/ffi/utils.dart | 3 + 8 files changed, 177 insertions(+), 146 deletions(-) create mode 100644 moor/lib/src/vm/ffi/utils.dart diff --git a/moor/lib/src/vm/api/database.dart b/moor/lib/src/vm/api/database.dart index b64aba38..97ad69de 100644 --- a/moor/lib/src/vm/api/database.dart +++ b/moor/lib/src/vm/api/database.dart @@ -5,9 +5,10 @@ import 'dart:typed_data'; import 'package:collection/collection.dart'; import 'package:moor/src/vm/bindings/constants.dart'; -import 'package:moor/src/vm/bindings/types.dart'; +import 'package:moor/src/vm/bindings/types.dart' as types; import 'package:moor/src/vm/bindings/bindings.dart'; import 'package:moor/src/vm/ffi/blob.dart'; +import 'package:moor/src/vm/ffi/utils.dart'; part 'errors.dart'; part 'prepared_statement.dart'; @@ -16,7 +17,7 @@ part 'result.dart'; const _openingFlags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE; class Database { - final DatabasePointer _db; + final Pointer _db; final List _preparedStmt = []; bool _isClosed = false; @@ -31,12 +32,12 @@ class Database { /// Opens an sqlite3 database from a filename. factory Database.open(String fileName) { - final dbOut = allocate(); - final pathC = CString.allocate(fileName); + final dbOut = Pointer>.allocate(); + final pathC = CBlob.allocateString(fileName); final resultCode = - bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, fromAddress(0)); - final dbPointer = dbOut.load(); + bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, nullptr.cast()); + final dbPointer = dbOut.load>(); dbOut.free(); pathC.free(); @@ -85,20 +86,20 @@ class Database { /// error occurs while executing. void execute(String sql) { _ensureOpen(); - final sqlPtr = CString.allocate(sql); - final errorOut = allocate(); + final sqlPtr = CBlob.allocateString(sql); + final errorOut = Pointer>.allocate(); - final result = bindings.sqlite3_exec( - _db, sqlPtr, fromAddress(0), fromAddress(0), errorOut); + final result = + bindings.sqlite3_exec(_db, sqlPtr, nullptr, nullptr, errorOut); sqlPtr.free(); - final errorPtr = errorOut.load(); + final errorPtr = errorOut.load>(); errorOut.free(); String errorMsg; - if (errorPtr != null) { - errorMsg = CString.fromC(errorPtr.cast()); + if (!isNullPointer(errorPtr)) { + errorMsg = errorPtr.load().readString(); // the message was allocated from sqlite, we need to free it bindings.sqlite3_free(errorPtr.cast()); } @@ -112,14 +113,14 @@ class Database { PreparedStatement prepare(String sql) { _ensureOpen(); - final stmtOut = allocate(); - final sqlPtr = CString.allocate(sql); + final stmtOut = Pointer>.allocate(); + final sqlPtr = CBlob.allocateString(sql); final resultCode = - bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, fromAddress(0)); + bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, nullptr.cast()); sqlPtr.free(); - final stmt = stmtOut.load(); + final stmt = stmtOut.load>(); stmtOut.free(); if (resultCode != Errors.SQLITE_OK) { diff --git a/moor/lib/src/vm/api/errors.dart b/moor/lib/src/vm/api/errors.dart index 59514d4c..10987f40 100644 --- a/moor/lib/src/vm/api/errors.dart +++ b/moor/lib/src/vm/api/errors.dart @@ -6,11 +6,17 @@ class SqliteException implements Exception { SqliteException(this.message, [this.explanation]); - factory SqliteException._fromErrorCode(DatabasePointer db, [int code]) { - final dbMessage = CString.fromC(bindings.sqlite3_errmsg(db).cast()); + factory SqliteException._fromErrorCode(Pointer db, + [int code]) { + // We don't need to free the pointer returned by sqlite3_errmsg: "Memory to + // hold the error message string is managed internally. The application does + // not need to worry about freeing the result." + // https://www.sqlite.org/c3ref/errcode.html + final dbMessage = bindings.sqlite3_errmsg(db).load().readString(); + String explanation; if (code != null) { - explanation = CString.fromC(bindings.sqlite3_errstr(code).cast()); + explanation = bindings.sqlite3_errstr(code).load().readString(); } return SqliteException(dbMessage, explanation); diff --git a/moor/lib/src/vm/api/prepared_statement.dart b/moor/lib/src/vm/api/prepared_statement.dart index b528a781..5b06d476 100644 --- a/moor/lib/src/vm/api/prepared_statement.dart +++ b/moor/lib/src/vm/api/prepared_statement.dart @@ -1,7 +1,7 @@ part of 'database.dart'; class PreparedStatement { - final StatementPointer _stmt; + final Pointer _stmt; final Database _db; bool _closed = false; @@ -40,7 +40,8 @@ class PreparedStatement { for (var i = 0; i < columnCount; i++) { // name pointer doesn't need to be disposed, that happens when we finalize - names[i] = CString.fromC(bindings.sqlite3_column_name(_stmt, i).cast()); + names[i] = + bindings.sqlite3_column_name(_stmt, i).load().readString(); } while (_step() == Errors.SQLITE_ROW) { @@ -58,12 +59,16 @@ class PreparedStatement { case Types.SQLITE_FLOAT: return bindings.sqlite3_column_double(_stmt, index); case Types.SQLITE_TEXT: - return CString.fromC(bindings.sqlite3_column_text(_stmt, index).cast()); + return bindings + .sqlite3_column_text(_stmt, index) + .load() + .readString(); case Types.SQLITE_BLOB: final length = bindings.sqlite3_column_bytes(_stmt, index); - final data = - CBlob.fromC(bindings.sqlite3_column_blob(_stmt, index), length); - return data; + return bindings + .sqlite3_column_blob(_stmt, index) + .load() + .read(length); case Types.SQLITE_NULL: default: return null; @@ -107,18 +112,17 @@ class PreparedStatement { } else if (param is num) { bindings.sqlite3_bind_double(_stmt, i, param.toDouble()); } else if (param is String) { - final ptr = CString.allocate(param); + final ptr = CBlob.allocateString(param); _allocatedWhileBinding.add(ptr); - bindings.sqlite3_bind_text(_stmt, i, ptr, -1, fromAddress(0)); + bindings.sqlite3_bind_text(_stmt, i, ptr, -1, nullptr); } else if (param is Uint8List) { // todo we just have a null pointer param.isEmpty. I guess we have // to use sqlite3_bind_zeroblob for that? final ptr = CBlob.allocate(param); _allocatedWhileBinding.add(ptr); - bindings.sqlite3_bind_blob( - _stmt, i, ptr, param.length, fromAddress(0)); + bindings.sqlite3_bind_blob(_stmt, i, ptr, param.length, nullptr); } } } diff --git a/moor/lib/src/vm/bindings/bindings.dart b/moor/lib/src/vm/bindings/bindings.dart index 9d13e022..6df0a945 100644 --- a/moor/lib/src/vm/bindings/bindings.dart +++ b/moor/lib/src/vm/bindings/bindings.dart @@ -15,72 +15,82 @@ import 'types.dart'; class _SQLiteBindings { DynamicLibrary sqlite; - int Function(CString filename, Pointer databaseOut, - int flags, CString vfs) sqlite3_open_v2; + int Function(Pointer filename, Pointer> databaseOut, + int flags, Pointer vfs) sqlite3_open_v2; - int Function(DatabasePointer database) sqlite3_close_v2; + int Function(Pointer database) sqlite3_close_v2; void Function(Pointer ptr) sqlite3_free; int Function( - DatabasePointer database, - CString query, + Pointer database, + Pointer query, int nbytes, - Pointer statementOut, - Pointer tail) sqlite3_prepare_v2; + Pointer> statementOut, + Pointer> tail) sqlite3_prepare_v2; int Function( - DatabasePointer database, - CString query, + Pointer database, + Pointer query, Pointer callback, Pointer cbFirstArg, - Pointer errorMsgOut, + Pointer> errorMsgOut, ) sqlite3_exec; - int Function(StatementPointer statement) sqlite3_step; + int Function(Pointer statement) sqlite3_step; - int Function(StatementPointer statement) sqlite3_reset; + int Function(Pointer statement) sqlite3_reset; - int Function(StatementPointer statement) sqlite3_finalize; + int Function(Pointer statement) sqlite3_finalize; - int Function(StatementPointer statement) sqlite3_column_count; + int Function(Pointer statement) sqlite3_column_count; - CString Function(StatementPointer statement, int columnIndex) + Pointer Function(Pointer statement, int columnIndex) sqlite3_column_name; - CString Function(StatementPointer statement, int columnIndex) + Pointer Function(Pointer statement, int columnIndex) sqlite3_column_decltype; - int Function(StatementPointer statement, int columnIndex) sqlite3_column_type; + int Function(Pointer statement, int columnIndex) + sqlite3_column_type; - ValuePointer Function(StatementPointer statement, int columnIndex) + Pointer Function(Pointer statement, int columnIndex) sqlite3_column_value; - double Function(StatementPointer statement, int columnIndex) + double Function(Pointer statement, int columnIndex) sqlite3_column_double; - int Function(StatementPointer statement, int columnIndex) sqlite3_column_int; - CString Function(StatementPointer statement, int columnIndex) + int Function(Pointer statement, int columnIndex) + sqlite3_column_int; + Pointer Function(Pointer statement, int columnIndex) sqlite3_column_text; - CBlob Function(StatementPointer statement, int columnIndex) + Pointer Function(Pointer statement, int columnIndex) sqlite3_column_blob; /// Returns the amount of bytes to read when using [sqlite3_column_blob]. - int Function(StatementPointer statement, int columnIndex) + int Function(Pointer statement, int columnIndex) sqlite3_column_bytes; - int Function(DatabasePointer db) sqlite3_changes; - int Function(DatabasePointer db) sqlite3_last_insert_rowid; + int Function(Pointer db) sqlite3_changes; + int Function(Pointer db) sqlite3_last_insert_rowid; - CString Function(int code) sqlite3_errstr; - CString Function(DatabasePointer database) sqlite3_errmsg; + Pointer Function(int code) sqlite3_errstr; + Pointer Function(Pointer database) sqlite3_errmsg; - int Function(StatementPointer statement, int columnIndex, double value) + int Function(Pointer statement, int columnIndex, double value) sqlite3_bind_double; - int Function(StatementPointer statement, int columnIndex, int value) + int Function(Pointer statement, int columnIndex, int value) sqlite3_bind_int; - int Function(StatementPointer statement, int columnIndex, CString value, - int minusOne, Pointer disposeCb) sqlite3_bind_text; - int Function(StatementPointer statement, int columnIndex, CBlob value, - int length, Pointer disposeCb) sqlite3_bind_blob; - int Function(StatementPointer statement, int columnIndex) sqlite3_bind_null; + int Function( + Pointer statement, + int columnIndex, + Pointer value, + int minusOne, + Pointer disposeCb) sqlite3_bind_text; + int Function( + Pointer statement, + int columnIndex, + Pointer value, + int length, + Pointer disposeCb) sqlite3_bind_blob; + int Function(Pointer statement, int columnIndex) sqlite3_bind_null; _SQLiteBindings() { sqlite = dlopenPlatformSpecific('sqlite3'); diff --git a/moor/lib/src/vm/bindings/signatures.dart b/moor/lib/src/vm/bindings/signatures.dart index 8e834f30..6f0a0930 100644 --- a/moor/lib/src/vm/bindings/signatures.dart +++ b/moor/lib/src/vm/bindings/signatures.dart @@ -8,78 +8,88 @@ import '../ffi/blob.dart'; import 'types.dart'; -typedef sqlite3_open_v2_native_t = Int32 Function( - CString filename, Pointer ppDb, Int32 flags, CString vfs); +typedef sqlite3_open_v2_native_t = Int32 Function(Pointer filename, + Pointer> ppDb, Int32 flags, Pointer vfs); -typedef sqlite3_close_v2_native_t = Int32 Function(DatabasePointer database); +typedef sqlite3_close_v2_native_t = Int32 Function(Pointer database); typedef sqlite3_free_native = Void Function(Pointer pointer); typedef sqlite3_prepare_v2_native_t = Int32 Function( - DatabasePointer database, - CString query, + Pointer database, + Pointer query, Int32 nbytes, - Pointer statementOut, - Pointer tail); + Pointer> statementOut, + Pointer> tail); typedef sqlite3_exec_native = Int32 Function( - DatabasePointer database, - CString query, + Pointer database, + Pointer query, Pointer callback, Pointer firstCbArg, - Pointer errorOut); + Pointer> errorOut); -typedef sqlite3_step_native_t = Int32 Function(StatementPointer statement); +typedef sqlite3_step_native_t = Int32 Function(Pointer statement); -typedef sqlite3_reset_native_t = Int32 Function(StatementPointer statement); +typedef sqlite3_reset_native_t = Int32 Function(Pointer statement); -typedef sqlite3_finalize_native_t = Int32 Function(StatementPointer statement); +typedef sqlite3_finalize_native_t = Int32 Function( + Pointer statement); -typedef sqlite3_errstr_native_t = CString Function(Int32 error); +typedef sqlite3_errstr_native_t = Pointer Function(Int32 error); -typedef sqlite3_errmsg_native_t = CString Function(DatabasePointer database); +typedef sqlite3_errmsg_native_t = Pointer Function( + Pointer database); typedef sqlite3_column_count_native_t = Int32 Function( - StatementPointer statement); + Pointer statement); -typedef sqlite3_column_name_native_t = CString Function( - StatementPointer statement, Int32 columnIndex); +typedef sqlite3_column_name_native_t = Pointer Function( + Pointer statement, Int32 columnIndex); -typedef sqlite3_column_decltype_native_t = CString Function( - StatementPointer statement, Int32 columnIndex); +typedef sqlite3_column_decltype_native_t = Pointer Function( + Pointer statement, Int32 columnIndex); typedef sqlite3_column_type_native_t = Int32 Function( - StatementPointer statement, Int32 columnIndex); + Pointer statement, Int32 columnIndex); -typedef sqlite3_column_value_native_t = ValuePointer Function( - StatementPointer statement, Int32 columnIndex); +typedef sqlite3_column_value_native_t = Pointer Function( + Pointer statement, Int32 columnIndex); typedef sqlite3_column_double_native_t = Double Function( - StatementPointer statement, Int32 columnIndex); + Pointer statement, Int32 columnIndex); typedef sqlite3_column_int_native_t = Int32 Function( - StatementPointer statement, Int32 columnIndex); + Pointer statement, Int32 columnIndex); -typedef sqlite3_column_text_native_t = CString Function( - StatementPointer statement, Int32 columnIndex); +typedef sqlite3_column_text_native_t = Pointer Function( + Pointer statement, Int32 columnIndex); -typedef sqlite3_column_blob_native_t = CBlob Function( - StatementPointer statement, Int32 columnIndex); +typedef sqlite3_column_blob_native_t = Pointer Function( + Pointer statement, Int32 columnIndex); typedef sqlite3_column_bytes_native_t = Int32 Function( - StatementPointer statement, Int32 columnIndex); + Pointer statement, Int32 columnIndex); -typedef sqlite3_changes_native = Int32 Function(DatabasePointer database); +typedef sqlite3_changes_native = Int32 Function(Pointer database); typedef sqlite3_last_insert_rowid_native = Int64 Function( - DatabasePointer database); + Pointer database); typedef sqlite3_bind_double_native = Int32 Function( - StatementPointer statement, Int32 columnIndex, Double value); + Pointer statement, Int32 columnIndex, Double value); typedef sqlite3_bind_int_native = Int32 Function( - StatementPointer statement, Int32 columnIndex, Int32 value); -typedef sqlite3_bind_text_native = Int32 Function(StatementPointer statement, - Int32 columnIndex, CString value, Int32 length, Pointer callback); -typedef sqlite3_bind_blob_native = Int32 Function(StatementPointer statement, - Int32 columnIndex, CBlob value, Int32 length, Pointer callback); + Pointer statement, Int32 columnIndex, Int32 value); +typedef sqlite3_bind_text_native = Int32 Function( + Pointer statement, + Int32 columnIndex, + Pointer value, + Int32 length, + Pointer callback); +typedef sqlite3_bind_blob_native = Int32 Function( + Pointer statement, + Int32 columnIndex, + Pointer value, + Int32 length, + Pointer callback); typedef sqlite3_bind_null_native = Int32 Function( - StatementPointer statement, Int32 columnIndex); + Pointer statement, Int32 columnIndex); diff --git a/moor/lib/src/vm/bindings/types.dart b/moor/lib/src/vm/bindings/types.dart index b3848b27..dc5efb50 100644 --- a/moor/lib/src/vm/bindings/types.dart +++ b/moor/lib/src/vm/bindings/types.dart @@ -6,8 +6,6 @@ import 'dart:ffi'; // ignore_for_file: comment_references -class FunctionPointer extends Pointer {} - /// Database Connection Handle /// /// Each open SQLite database is represented by a pointer to an instance of @@ -17,7 +15,7 @@ class FunctionPointer extends Pointer {} /// is its destructor. There are many other interfaces (such as /// [sqlite3_prepare_v2()], [sqlite3_create_function()], and /// [sqlite3_busy_timeout()] to name but three) that are methods on an -class DatabasePointer extends Pointer {} +class Database extends Struct {} /// SQL Statement Object /// @@ -40,7 +38,7 @@ class DatabasePointer extends Pointer {} /// /// Refer to documentation on individual methods above for additional /// information. -class StatementPointer extends Pointer {} +class Statement extends Struct {} /// Dynamically Typed Value Object /// @@ -76,4 +74,4 @@ class StatementPointer extends Pointer {} /// [sqlite3_result_value()] and [sqlite3_bind_value()]. /// The [sqlite3_value_blob | sqlite3_value_type()] family of /// interfaces require protected sqlite3_value objects. -class ValuePointer extends Pointer {} +class Value extends Struct {} diff --git a/moor/lib/src/vm/ffi/blob.dart b/moor/lib/src/vm/ffi/blob.dart index b29ea671..f1d9bce5 100644 --- a/moor/lib/src/vm/ffi/blob.dart +++ b/moor/lib/src/vm/ffi/blob.dart @@ -3,53 +3,52 @@ import 'dart:ffi'; import 'dart:typed_data'; -/// Pointer to arbitrary blobs that aren't null-terminated. -class CBlob extends Pointer { - /// Allocate a [CBlob] not managed in and populates it with [dartBlob]. - factory CBlob.allocate(Uint8List dartBlob) { - if (dartBlob.isEmpty) { - return fromAddress(0); - } +import 'package:moor/src/vm/ffi/utils.dart'; - final ptr = allocate(count: dartBlob.length); - for (var i = 0; i < dartBlob.length; ++i) { - ptr.elementAt(i).store(dartBlob[i]); +/// Pointer to arbitrary blobs in C. +class CBlob extends Struct { + @Uint8() + int data; + + static Pointer allocate(Uint8List blob) { + final str = Pointer.allocate(count: blob.length); + for (var i = 0; i < blob.length; i++) { + str.elementAt(i).load().data = blob[i]; } - return ptr.cast(); + return str; } - /// Read the string from C memory into Dart. - static Uint8List fromC(CBlob str, int length) { - if (str == null) return null; - assert(length >= 0); - - final units = Uint8List(length); - for (var i = 0; i < length; ++i) { - units[i] = str.elementAt(i).load(); - } - - return units; - } -} - -/// A null-terminated C string. -class CString extends Pointer { - /// Allocate a [CString] not managed in and populates it with [string]. - factory CString.allocate(String string) { + /// Allocates a 0-terminated string, encoded as utf8 and read from the + /// [string]. + static Pointer allocateString(String string) { final encoded = utf8.encode(string); final data = Uint8List(encoded.length + 1) // already filled with zeroes ..setAll(0, encoded); - - return CBlob.allocate(data).cast(); + return CBlob.allocate(data); } - /// Read the string from C memory into Dart. - static String fromC(CBlob str) { - if (str == null) return null; - var len = 0; - while (str.elementAt(++len).load() != 0) {} + /// Reads [bytesToRead] bytes from the current position. + Uint8List read(int bytesToRead) { + assert(bytesToRead >= 0); + final str = addressOf; + if (isNullPointer(str)) return null; - final list = CBlob.fromC(str, len); - return utf8.decode(list); + final blob = Uint8List(bytesToRead); + for (var i = 0; i < bytesToRead; ++i) { + blob[i] = str.elementAt(i).load().data; + } + return blob; + } + + /// Reads a 0-terminated string, decoded with utf8. + String readString() { + final str = addressOf; + if (isNullPointer(str)) return null; + + var len = 0; + while (str.elementAt(++len).load().data != 0) {} + + final units = read(len); + return utf8.decode(units); } } diff --git a/moor/lib/src/vm/ffi/utils.dart b/moor/lib/src/vm/ffi/utils.dart new file mode 100644 index 00000000..518d5841 --- /dev/null +++ b/moor/lib/src/vm/ffi/utils.dart @@ -0,0 +1,3 @@ +import 'dart:ffi'; + +bool isNullPointer(Pointer ptr) => ptr == nullptr; From 78bb23a7d1b36a559a78319fa297a30074635d23 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 28 Aug 2019 17:46:52 +0200 Subject: [PATCH 029/117] Raise SDK constraints to 2.5.0 for FFI version --- .cirrus.yml | 4 ++-- moor/pubspec.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 0338e46e..4cb8e831 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,6 @@ # Run tasks with the dart SDK installed by default -container: - image: "google/dart:latest" +container: # todo set back to latest + image: "google/dart:dev" # We're currently not running tests with coverage because the free cirrus containers run out of memory :( diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index aaf3dd7c..8a4595cd 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -9,7 +9,7 @@ authors: maintainer: Simon Binder (@simolus3) environment: - sdk: '>=2.2.2 <3.0.0' + sdk: '>=2.5.0-dev <3.0.0' dependencies: meta: ^1.0.0 From 4e83c0275b1d0d3913d4028a1b2d46d6f3c5da52 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 28 Aug 2019 21:52:15 +0200 Subject: [PATCH 030/117] Generate constructor for companions with @required fields --- moor/example/example.g.dart | 24 ++++++++++ moor/test/data/tables/custom_tables.g.dart | 20 ++++++++ moor/test/data/tables/todos.g.dart | 35 ++++++++++++++ .../lib/src/model/specified_column.dart | 8 ++++ .../src/writer/update_companion_writer.dart | 47 +++++++++++++++++++ 5 files changed, 134 insertions(+) diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index 31184953..31dd4ec8 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -76,6 +76,10 @@ class CategoriesCompanion extends UpdateCompanion { this.id = const Value.absent(), this.description = const Value.absent(), }); + CategoriesCompanion.insert({ + this.id = const Value.absent(), + this.description = const Value.absent(), + }); CategoriesCompanion copyWith({Value id, Value description}) { return CategoriesCompanion( id: id ?? this.id, @@ -266,6 +270,13 @@ class RecipesCompanion extends UpdateCompanion { this.instructions = const Value.absent(), this.category = const Value.absent(), }); + RecipesCompanion.insert({ + this.id = const Value.absent(), + @required String title, + @required String instructions, + this.category = const Value.absent(), + }) : this.title = Value(title), + this.instructions = Value(instructions); RecipesCompanion copyWith( {Value id, Value title, @@ -482,6 +493,12 @@ class IngredientsCompanion extends UpdateCompanion { this.name = const Value.absent(), this.caloriesPer100g = const Value.absent(), }); + IngredientsCompanion.insert({ + this.id = const Value.absent(), + @required String name, + @required int caloriesPer100g, + }) : this.name = Value(name), + this.caloriesPer100g = Value(caloriesPer100g); IngredientsCompanion copyWith( {Value id, Value name, Value caloriesPer100g}) { return IngredientsCompanion( @@ -688,6 +705,13 @@ class IngredientInRecipesCompanion extends UpdateCompanion { this.ingredient = const Value.absent(), this.amountInGrams = const Value.absent(), }); + IngredientInRecipesCompanion.insert({ + @required int recipe, + @required int ingredient, + @required int amountInGrams, + }) : this.recipe = Value(recipe), + this.ingredient = Value(ingredient), + this.amountInGrams = Value(amountInGrams); IngredientInRecipesCompanion copyWith( {Value recipe, Value ingredient, Value amountInGrams}) { return IngredientInRecipesCompanion( diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index bdf39b2e..7a7b75c9 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -63,6 +63,9 @@ class NoIdsCompanion extends UpdateCompanion { const NoIdsCompanion({ this.payload = const Value.absent(), }); + NoIdsCompanion.insert({ + @required Uint8List payload, + }) : this.payload = Value(payload); NoIdsCompanion copyWith({Value payload}) { return NoIdsCompanion( payload: payload ?? this.payload, @@ -197,6 +200,10 @@ class WithDefaultsCompanion extends UpdateCompanion { this.a = const Value.absent(), this.b = const Value.absent(), }); + WithDefaultsCompanion.insert({ + this.a = const Value.absent(), + this.b = const Value.absent(), + }); WithDefaultsCompanion copyWith({Value a, Value b}) { return WithDefaultsCompanion( a: a ?? this.a, @@ -359,6 +366,11 @@ class WithConstraintsCompanion extends UpdateCompanion { this.b = const Value.absent(), this.c = const Value.absent(), }); + WithConstraintsCompanion.insert({ + this.a = const Value.absent(), + @required int b, + this.c = const Value.absent(), + }) : this.b = Value(b); WithConstraintsCompanion copyWith( {Value a, Value b, Value c}) { return WithConstraintsCompanion( @@ -535,6 +547,10 @@ class ConfigCompanion extends UpdateCompanion { this.configKey = const Value.absent(), this.configValue = const Value.absent(), }); + ConfigCompanion.insert({ + @required String configKey, + this.configValue = const Value.absent(), + }) : this.configKey = Value(configKey); ConfigCompanion copyWith( {Value configKey, Value configValue}) { return ConfigCompanion( @@ -694,6 +710,10 @@ class MytableCompanion extends UpdateCompanion { this.someid = const Value.absent(), this.sometext = const Value.absent(), }); + MytableCompanion.insert({ + this.someid = const Value.absent(), + this.sometext = const Value.absent(), + }); MytableCompanion copyWith({Value someid, Value sometext}) { return MytableCompanion( someid: someid ?? this.someid, diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 5d5bbce7..99f3a799 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -133,6 +133,13 @@ class TodosTableCompanion extends UpdateCompanion { this.targetDate = const Value.absent(), this.category = const Value.absent(), }); + TodosTableCompanion.insert({ + this.id = const Value.absent(), + this.title = const Value.absent(), + @required String content, + this.targetDate = const Value.absent(), + this.category = const Value.absent(), + }) : this.content = Value(content); TodosTableCompanion copyWith( {Value id, Value title, @@ -358,6 +365,10 @@ class CategoriesCompanion extends UpdateCompanion { this.id = const Value.absent(), this.description = const Value.absent(), }); + CategoriesCompanion.insert({ + this.id = const Value.absent(), + @required String description, + }) : this.description = Value(description); CategoriesCompanion copyWith({Value id, Value description}) { return CategoriesCompanion( id: id ?? this.id, @@ -569,6 +580,14 @@ class UsersCompanion extends UpdateCompanion { this.profilePicture = const Value.absent(), this.creationTime = const Value.absent(), }); + UsersCompanion.insert({ + this.id = const Value.absent(), + @required String name, + this.isAwesome = const Value.absent(), + @required Uint8List profilePicture, + this.creationTime = const Value.absent(), + }) : this.name = Value(name), + this.profilePicture = Value(profilePicture); UsersCompanion copyWith( {Value id, Value name, @@ -792,6 +811,11 @@ class SharedTodosCompanion extends UpdateCompanion { this.todo = const Value.absent(), this.user = const Value.absent(), }); + SharedTodosCompanion.insert({ + @required int todo, + @required int user, + }) : this.todo = Value(todo), + this.user = Value(user); SharedTodosCompanion copyWith({Value todo, Value user}) { return SharedTodosCompanion( todo: todo ?? this.todo, @@ -978,6 +1002,13 @@ class TableWithoutPKCompanion extends UpdateCompanion { this.someFloat = const Value.absent(), this.custom = const Value.absent(), }); + TableWithoutPKCompanion.insert({ + @required int notReallyAnId, + @required double someFloat, + @required MyCustomObject custom, + }) : this.notReallyAnId = Value(notReallyAnId), + this.someFloat = Value(someFloat), + this.custom = Value(custom); TableWithoutPKCompanion copyWith( {Value notReallyAnId, Value someFloat, @@ -1162,6 +1193,10 @@ class PureDefaultsCompanion extends UpdateCompanion { this.id = const Value.absent(), this.txt = const Value.absent(), }); + PureDefaultsCompanion.insert({ + this.id = const Value.absent(), + this.txt = const Value.absent(), + }); PureDefaultsCompanion copyWith({Value id, Value txt}) { return PureDefaultsCompanion( id: id ?? this.id, diff --git a/moor_generator/lib/src/model/specified_column.dart b/moor_generator/lib/src/model/specified_column.dart index 75d59b2e..6e63d6fc 100644 --- a/moor_generator/lib/src/model/specified_column.dart +++ b/moor_generator/lib/src/model/specified_column.dart @@ -143,6 +143,14 @@ class SpecifiedColumn { ColumnType.real: 'GeneratedRealColumn', }[type]; + /// Whether this column is required for insert statements, meaning that a + /// non-absent value must be provided for an insert statement to be valid. + bool get requiredDuringInsert { + final aliasForPk = type == ColumnType.integer && + features.any((f) => f is PrimaryKey || f is AutoIncrement); + return !nullable && defaultArgument == null && !aliasForPk; + } + /// The class inside the moor library that represents the same sql type as /// this column. String get sqlTypeName => sqlTypes[type]; diff --git a/moor_generator/lib/src/writer/update_companion_writer.dart b/moor_generator/lib/src/writer/update_companion_writer.dart index 3754fe98..9ed7f89b 100644 --- a/moor_generator/lib/src/writer/update_companion_writer.dart +++ b/moor_generator/lib/src/writer/update_companion_writer.dart @@ -1,3 +1,4 @@ +import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/state/session.dart'; @@ -12,6 +13,7 @@ class UpdateCompanionWriter { 'extends UpdateCompanion<${table.dartTypeName}> {\n'); _writeFields(buffer); _writeConstructor(buffer); + _writeInsertConstructor(buffer); _writeCopyWith(buffer); buffer.write('}\n'); @@ -34,6 +36,51 @@ class UpdateCompanionWriter { buffer.write('});\n'); } + /// Writes a special `.insert` constructor. All columns which may not be + /// absent during insert are marked `@required` here. Also, we don't need to + /// use value wrappers here - `Value.absent` simply isn't an option. + void _writeInsertConstructor(StringBuffer buffer) { + final requiredColumns = {}; + + // can't be constant because we use initializers (this.a = Value(a)). + // for a parameter a which is only potentially constant. + buffer.write('${table.updateCompanionName}.insert({'); + + // Say we had two required columns a and c, and an optional column b. + // .insert({ + // @required String a, + // this.b = const Value.absent(), + // @required String b}): this.a = Value(a), this.b = Value(b); + + for (var column in table.columns) { + final param = column.dartGetterName; + + if (column.requiredDuringInsert) { + requiredColumns.add(column); + + buffer.write('@required ${column.dartTypeName} $param,'); + } else { + buffer.write('this.$param = const Value.absent(),'); + } + } + buffer.write('})'); + + var first = true; + for (var required in requiredColumns) { + if (first) { + buffer.write(': '); + first = false; + } else { + buffer.write(', '); + } + + final param = required.dartGetterName; + buffer.write('this.$param = Value($param)'); + } + + buffer.write(';\n'); + } + void _writeCopyWith(StringBuffer buffer) { buffer.write('${table.updateCompanionName} copyWith({'); var first = true; From 6a046ec408824c92c75dbda55e61337c671cc12a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 28 Aug 2019 22:12:23 +0200 Subject: [PATCH 031/117] Explain why companions exist in the documentation --- .../en/docs/Getting started/writing_queries.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/content/en/docs/Getting started/writing_queries.md b/docs/content/en/docs/Getting started/writing_queries.md index a7d46414..384e75d4 100644 --- a/docs/content/en/docs/Getting started/writing_queries.md +++ b/docs/content/en/docs/Getting started/writing_queries.md @@ -88,6 +88,21 @@ Future feelingLazy() { __āš ļø Caution:__ If you don't explicitly add a `where` clause on updates or deletes, the statement will affect all rows in the table! +{{% alert title="Entries, companions - why do we need all of this?" %}} +You might have noticed that we used a `TodosCompanion` for the first update instead of +just passing a `TodoEntry`. Moor generates the `TodoEntry` class (also called _data +class_ for the table) to hold a __full__ row with all its data. For _partial_ data, +prefer to use companions. In the example above, we only set the the `category` column, +so we used a companion. +Why is that necessary? If a field was set to `null`, we wouldn't know whether we need +to set that column back to null in the database or if we should just leave it unchanged. +Fields in the companions have a special `Value.absent()` state which makes this explicit. + +Companions also have a special constructor for inserts - all columns which don't have +a default value and aren't nullable are marked `@required` on that constructor. This makes +companions easier to use for inserts because you know which fields to set. +{{% /alert %}} + ## Inserts You can very easily insert any valid object into tables. As some values can be absent (like default values that we don't have to set explicitly), we again use the From 2f8dc6d68ec697b89e3e7953c30482127bf6019f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 29 Aug 2019 15:04:39 +0200 Subject: [PATCH 032/117] Parse insert statements --- .../analysis/steps/reference_resolver.dart | 6 +- sqlparser/lib/src/ast/ast.dart | 5 ++ sqlparser/lib/src/ast/statements/insert.dart | 90 +++++++++++++++++++ sqlparser/lib/src/reader/parser/crud.dart | 69 ++++++++++++++ .../lib/src/reader/parser/expressions.dart | 15 ++++ sqlparser/lib/src/reader/parser/parser.dart | 7 +- sqlparser/lib/src/reader/tokenizer/token.dart | 6 ++ sqlparser/test/parser/insert_test.dart | 57 ++++++++++++ 8 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 sqlparser/lib/src/ast/statements/insert.dart create mode 100644 sqlparser/test/parser/insert_test.dart diff --git a/sqlparser/lib/src/analysis/steps/reference_resolver.dart b/sqlparser/lib/src/analysis/steps/reference_resolver.dart index 8090f91c..046a18e9 100644 --- a/sqlparser/lib/src/analysis/steps/reference_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/reference_resolver.dart @@ -8,6 +8,10 @@ class ReferenceResolver extends RecursiveVisitor { @override void visitReference(Reference e) { + if (e.resolved != null) { + return super.visitReference(e); + } + final scope = e.scope; if (e.tableName != null) { @@ -65,7 +69,7 @@ class ReferenceResolver extends RecursiveVisitor { @override void visitAggregateExpression(AggregateExpression e) { - if (e.windowName != null) { + if (e.windowName != null && e.resolved != null) { final resolved = e.scope.resolve(e.windowName); e.resolved = resolved; } diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index 86e970b9..b7b218f4 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -26,6 +26,7 @@ part 'schema/table_definition.dart'; part 'statements/create_table.dart'; part 'statements/delete.dart'; +part 'statements/insert.dart'; part 'statements/select.dart'; part 'statements/statement.dart'; part 'statements/update.dart'; @@ -133,6 +134,7 @@ abstract class AstNode { abstract class AstVisitor { T visitSelectStatement(SelectStatement e); T visitResultColumn(ResultColumn e); + T visitInsertStatement(InsertStatement e); T visitDeleteStatement(DeleteStatement e); T visitUpdateStatement(UpdateStatement e); T visitCreateTableStatement(CreateTableStatement e); @@ -248,6 +250,9 @@ class RecursiveVisitor extends AstVisitor { @override T visitSelectStatement(SelectStatement e) => visitChildren(e); + @override + T visitInsertStatement(InsertStatement e) => visitChildren(e); + @override T visitDeleteStatement(DeleteStatement e) => visitChildren(e); diff --git a/sqlparser/lib/src/ast/statements/insert.dart b/sqlparser/lib/src/ast/statements/insert.dart new file mode 100644 index 00000000..d7550fd9 --- /dev/null +++ b/sqlparser/lib/src/ast/statements/insert.dart @@ -0,0 +1,90 @@ +part of '../ast.dart'; + +enum InsertMode { + insert, + replace, + insertOrReplace, + insertOrRollback, + insertOrAbort, + insertOrFail, + insertOrIgnore +} + +class InsertStatement extends Statement with CrudStatement { + final InsertMode mode; + final TableReference table; + final List targetColumns; + final InsertSource source; + + // todo parse upsert clauses + + InsertStatement( + {this.mode = InsertMode.insert, + @required this.table, + @required this.targetColumns, + @required this.source}); + + @override + T accept(AstVisitor visitor) => visitor.visitInsertStatement(this); + + @override + Iterable get childNodes sync* { + yield table; + yield* targetColumns; + yield* source.childNodes; + } + + @override + bool contentEquals(InsertStatement other) { + return other.mode == mode && other.source.runtimeType == source.runtimeType; + } +} + +abstract class InsertSource { + Iterable get childNodes; + + const InsertSource(); + + T when( + {T Function(ValuesSource) isValues, + T Function(SelectInsertSource) isSelect, + T Function(DefaultValues) isDefaults}) { + if (this is ValuesSource) { + return isValues?.call(this as ValuesSource); + } else if (this is SelectInsertSource) { + return isSelect?.call(this as SelectInsertSource); + } else if (this is DefaultValues) { + return isDefaults?.call(this as DefaultValues); + } else { + throw StateError('Did not expect $runtimeType as InsertSource'); + } + } +} + +/// Uses a list of values for an insert statement (`VALUES (a, b, c)`). +class ValuesSource extends InsertSource { + final List values; + + ValuesSource(this.values); + + @override + Iterable get childNodes => values; +} + +/// Inserts the rows returned by [stmt]. +class SelectInsertSource extends InsertSource { + final SelectStatement stmt; + + SelectInsertSource(this.stmt); + + @override + Iterable get childNodes => [stmt]; +} + +/// Use `DEFAULT VALUES` for an insert statement. +class DefaultValues extends InsertSource { + const DefaultValues(); + + @override + final Iterable childNodes = const []; +} diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 9545eb84..e0ba177f 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -368,6 +368,75 @@ mixin CrudParser on ParserBase { or: failureMode, table: table, set: set, where: where); } + InsertStatement _insertStmt() { + if (!_match(const [TokenType.insert, TokenType.replace])) return null; + + final firstToken = _previous; + InsertMode insertMode; + if (_previous.type == TokenType.insert) { + // insert modes can have a failure clause (INSERT OR xxx) + if (_matchOne(TokenType.or)) { + const tokensToModes = { + TokenType.replace: InsertMode.insertOrReplace, + TokenType.rollback: InsertMode.insertOrRollback, + TokenType.abort: InsertMode.insertOrAbort, + TokenType.fail: InsertMode.insertOrFail, + TokenType.ignore: InsertMode.insertOrIgnore + }; + + if (_match(tokensToModes.keys)) { + insertMode = tokensToModes[_previous.type]; + } else { + _error( + 'After the INSERT OR, expected an insert mode (REPLACE, ROLLBACK, etc.)'); + } + } else { + insertMode = InsertMode.insert; + } + } else { + // is it wasn't an insert, it must have been a replace + insertMode = InsertMode.replace; + } + assert(insertMode != null); + _consume(TokenType.into, 'Expected INSERT INTO'); + + final table = _tableReference(); + final targetColumns = []; + + if (_matchOne(TokenType.leftParen)) { + do { + final columnRef = _consumeIdentifier('Expected a column'); + targetColumns.add(Reference(columnName: columnRef.identifier)); + } while (_matchOne(TokenType.comma)); + + _consume(TokenType.rightParen, + 'Expected clpsing parenthesis after column list'); + } + final source = _insertSource(); + + return InsertStatement( + mode: insertMode, + table: table, + targetColumns: targetColumns, + source: source, + )..setSpan(firstToken, _previous); + } + + InsertSource _insertSource() { + if (_matchOne(TokenType.$values)) { + final values = []; + do { + values.add(_consumeTuple()); + } while (_matchOne(TokenType.comma)); + return ValuesSource(values); + } else if (_matchOne(TokenType.$default)) { + _consume(TokenType.$values, 'Expected DEFAULT VALUES'); + return const DefaultValues(); + } else { + return SelectInsertSource(select()); + } + } + @override WindowDefinition _windowDefinition() { _consume(TokenType.leftParen, 'Expected opening parenthesis'); diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index 61227a0a..594f0a6e 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -391,4 +391,19 @@ mixin ExpressionParser on ParserBase { windowName: windowName, )..setSpan(name, _previous); } + + TupleExpression _consumeTuple() { + final firstToken = + _consume(TokenType.leftParen, 'Expected opening parenthesis for tuple'); + final expressions = []; + + do { + expressions.add(expression()); + } while (_matchOne(TokenType.comma)); + + _consume(TokenType.rightParen, 'Expected right parenthesis to close tuple'); + + return TupleExpression(expressions: expressions) + ..setSpan(firstToken, _previous); + } } diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 8fc1de35..05a39cbd 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -129,6 +129,7 @@ abstract class ParserBase { // Common operations that we are referenced very often Expression expression(); + TupleExpression _consumeTuple(); /// Parses a [SelectStatement], or returns null if there is no select token /// after the current position. @@ -153,7 +154,11 @@ class Parser extends ParserBase Statement statement({bool expectEnd = true}) { final first = _peek; - final stmt = select() ?? _deleteStmt() ?? _update() ?? _createTable(); + final stmt = select() ?? + _deleteStmt() ?? + _update() ?? + _insertStmt() ?? + _createTable(); if (stmt == null) { _error('Expected a sql statement to start here'); diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index b7c7251d..ebc12054 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -57,6 +57,8 @@ enum TokenType { select, delete, update, + insert, + into, distinct, all, from, @@ -124,6 +126,7 @@ enum TokenType { unique, check, $default, + $values, conflict, references, cascade, @@ -141,6 +144,8 @@ enum TokenType { const Map keywords = { 'SELECT': TokenType.select, + 'INSERT': TokenType.insert, + 'INTO': TokenType.into, 'COLLATE': TokenType.collate, 'DISTINCT': TokenType.distinct, 'UPDATE': TokenType.update, @@ -228,6 +233,7 @@ const Map keywords = { 'OTHERS': TokenType.others, 'TIES': TokenType.ties, 'WINDOW': TokenType.window, + 'VALUES': TokenType.$values, }; const Map moorKeywords = { diff --git a/sqlparser/test/parser/insert_test.dart b/sqlparser/test/parser/insert_test.dart new file mode 100644 index 00000000..75580cb5 --- /dev/null +++ b/sqlparser/test/parser/insert_test.dart @@ -0,0 +1,57 @@ +import 'package:test/test.dart'; +import 'package:sqlparser/sqlparser.dart'; + +import 'utils.dart'; + +void main() { + test('parses insert statements', () { + testStatement( + 'INSERT OR REPLACE INTO tbl (a, b, c) VALUES (d, e, f)', + InsertStatement( + mode: InsertMode.insertOrReplace, + table: TableReference('tbl', null), + targetColumns: [ + Reference(columnName: 'a'), + Reference(columnName: 'b'), + Reference(columnName: 'c'), + ], + source: ValuesSource([ + TupleExpression(expressions: [ + Reference(columnName: 'd'), + Reference(columnName: 'e'), + Reference(columnName: 'f'), + ]), + ]), + ), + ); + }); + + test('insert statement with default values', () { + testStatement( + 'INSERT INTO tbl DEFAULT VALUES', + InsertStatement( + mode: InsertMode.insert, + table: TableReference('tbl', null), + targetColumns: const [], + source: const DefaultValues(), + ), + ); + }); + + test('insert statement with select as source', () { + testStatement( + 'REPLACE INTO tbl SELECT * FROM tbl', + InsertStatement( + mode: InsertMode.replace, + table: TableReference('tbl', null), + targetColumns: const [], + source: SelectInsertSource( + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + ), + ), + ), + ); + }); +} From dd8b4ab03a7f4892849b1b138bb12b108af4e530 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 29 Aug 2019 15:32:45 +0200 Subject: [PATCH 033/117] Infer types for insert statements --- sqlparser/README.md | 2 +- sqlparser/lib/src/analysis/analysis.dart | 2 +- .../src/analysis/steps/column_resolver.dart | 8 +++++ .../lib/src/analysis/steps/type_resolver.dart | 30 +++++++++++++++++-- .../lib/src/analysis/types/resolver.dart | 5 ++++ .../lib/src/ast/expressions/reference.dart | 2 ++ sqlparser/lib/src/ast/statements/insert.dart | 9 ++++++ .../test/analysis/type_resolver_test.dart | 13 ++++++++ 8 files changed, 67 insertions(+), 4 deletions(-) diff --git a/sqlparser/README.md b/sqlparser/README.md index 3c9f23cf..728977b3 100644 --- a/sqlparser/README.md +++ b/sqlparser/README.md @@ -65,10 +65,10 @@ package to generate type-safe methods from sql. Most on this list is just not supported yet because I didn't found a use case for them yet. If you need them, just leave an issue and I'll try to implement them soon. -- For now, `INSERT` statements are not supported, but they will be soon. - Compound select statements (`UNION` / `INTERSECT`) are not supported yet - Common table expressions are not supported - Some advanced expressions, like `CAST`s aren't supported yet. +- An `UPSERT` clause is not yet supported on insert statements If you run into parsing errors with what you think is valid sql, please create an issue. diff --git a/sqlparser/lib/src/analysis/analysis.dart b/sqlparser/lib/src/analysis/analysis.dart index b52b5ea2..af770458 100644 --- a/sqlparser/lib/src/analysis/analysis.dart +++ b/sqlparser/lib/src/analysis/analysis.dart @@ -1,4 +1,4 @@ -import 'dart:math'; +import 'dart:math' show min, max; import 'package:meta/meta.dart'; import 'package:source_span/source_span.dart'; diff --git a/sqlparser/lib/src/analysis/steps/column_resolver.dart b/sqlparser/lib/src/analysis/steps/column_resolver.dart index 3d38d9c5..1f00ae02 100644 --- a/sqlparser/lib/src/analysis/steps/column_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/column_resolver.dart @@ -22,6 +22,14 @@ class ColumnResolver extends RecursiveVisitor { visitChildren(e); } + @override + void visitInsertStatement(InsertStatement e) { + final table = _resolveTableReference(e.table); + visitChildren(e); + e.scope.availableColumns = table.resolvedColumns; + visitChildren(e); + } + @override void visitDeleteStatement(DeleteStatement e) { final table = _resolveTableReference(e.from); diff --git a/sqlparser/lib/src/analysis/steps/type_resolver.dart b/sqlparser/lib/src/analysis/steps/type_resolver.dart index 154040d9..987cce4b 100644 --- a/sqlparser/lib/src/analysis/steps/type_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/type_resolver.dart @@ -1,7 +1,8 @@ part of '../analysis.dart'; -/// Resolves the type of columns in a select statement and the type of -/// expressions appearing in a select statement. +/// Resolves types for all nodes in the AST which can have a type. This includes +/// expressions, variables and so on. For select statements, we also try to +/// figure out what types they return. class TypeResolvingVisitor extends RecursiveVisitor { final AnalysisContext context; TypeResolver get types => context.types; @@ -19,4 +20,29 @@ class TypeResolvingVisitor extends RecursiveVisitor { super.visitChildren(e); } + + @override + void visitInsertStatement(InsertStatement e) { + // resolve target columns - this is easy, as we should have the table + // structure available. + e.targetColumns.forEach(types.resolveExpression); + + // if the insert statement has a VALUES source, we can now infer the type + // for those expressions by comparing with the target column. + if (e.source is ValuesSource) { + final targetTypes = e.resolvedTargetColumns.map(context.typeOf).toList(); + final source = e.source as ValuesSource; + + for (var tuple in source.values) { + final expressions = tuple.expressions; + for (var i = 0; i < min(expressions.length, targetTypes.length); i++) { + if (i < targetTypes.length) { + context.types.markResult(expressions[i], targetTypes[i]); + } + } + } + } + + visitChildren(e); + } } diff --git a/sqlparser/lib/src/analysis/types/resolver.dart b/sqlparser/lib/src/analysis/types/resolver.dart index 9ba8dab7..4a5cc35e 100644 --- a/sqlparser/lib/src/analysis/types/resolver.dart +++ b/sqlparser/lib/src/analysis/types/resolver.dart @@ -29,6 +29,11 @@ class TypeResolver { return calculated; } + /// Manually writes the [result] for the [Typeable] [t]. + void markResult(Typeable t, ResolveResult result) { + _results.putIfAbsent(t, () => result); + } + ResolveResult resolveOrInfer(Typeable t) { if (t is Column) { return resolveColumn(t); diff --git a/sqlparser/lib/src/ast/expressions/reference.dart b/sqlparser/lib/src/ast/expressions/reference.dart index bba0a6d4..66b94d73 100644 --- a/sqlparser/lib/src/ast/expressions/reference.dart +++ b/sqlparser/lib/src/ast/expressions/reference.dart @@ -11,6 +11,8 @@ class Reference extends Expression with ReferenceOwner { final String tableName; final String columnName; + Column get resolvedColumn => resolved as Column; + Reference({this.tableName, this.columnName}); @override diff --git a/sqlparser/lib/src/ast/statements/insert.dart b/sqlparser/lib/src/ast/statements/insert.dart index d7550fd9..f4385526 100644 --- a/sqlparser/lib/src/ast/statements/insert.dart +++ b/sqlparser/lib/src/ast/statements/insert.dart @@ -16,6 +16,15 @@ class InsertStatement extends Statement with CrudStatement { final List targetColumns; final InsertSource source; + List get resolvedTargetColumns { + if (targetColumns.isNotEmpty) { + return targetColumns.map((c) => c.resolvedColumn).toList(); + } else { + // no columns declared - assume all columns from the table + return table.resultSet?.resolvedColumns; + } + } + // todo parse upsert clauses InsertStatement( diff --git a/sqlparser/test/analysis/type_resolver_test.dart b/sqlparser/test/analysis/type_resolver_test.dart index f820e5f2..0e1a2f1d 100644 --- a/sqlparser/test/analysis/type_resolver_test.dart +++ b/sqlparser/test/analysis/type_resolver_test.dart @@ -46,6 +46,19 @@ void main() { }); }); + test('handles VALUES clause in insert statements', () { + final engine = SqlEngine()..registerTable(demoTable); + final context = engine.analyze('INSERT INTO demo VALUES (?, ?), (?, ?)'); + + final variables = + context.root.allDescendants.whereType().toList(); + + expect(context.typeOf(variables[0]), ResolveResult(id.type)); + expect(context.typeOf(variables[1]), ResolveResult(content.type)); + expect(context.typeOf(variables[2]), ResolveResult(id.type)); + expect(context.typeOf(variables[3]), ResolveResult(content.type)); + }); + test('handles nth_value', () { final ctx = SqlEngine().analyze("SELECT nth_value('string', ?1) = ?2"); final variables = ctx.root.allDescendants.whereType().iterator; From a4b256f8a555c39713408dd79bd67437f453932d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 29 Aug 2019 15:52:29 +0200 Subject: [PATCH 034/117] Support custom insert statements at runtime --- moor/lib/src/runtime/database.dart | 32 +++++++++++++++++++++++++++--- moor/test/custom_queries_test.dart | 24 +++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/moor/lib/src/runtime/database.dart b/moor/lib/src/runtime/database.dart index 034cc53d..4427d40b 100644 --- a/moor/lib/src/runtime/database.dart +++ b/moor/lib/src/runtime/database.dart @@ -12,6 +12,9 @@ import 'package:moor/src/runtime/statements/update.dart'; const _zoneRootUserKey = #DatabaseConnectionUser; +typedef _CustomWriter = Future Function( + QueryExecutor e, String sql, List vars); + /// Class that runs queries to a subset of all available queries in a database. /// /// This comes in handy to structure large amounts of database code better: The @@ -174,20 +177,43 @@ mixin QueryEngine on DatabaseConnectionUser { @visibleForTesting Future customUpdate(String query, {List variables = const [], Set updates}) async { + return _customWrite(query, variables, updates, (executor, sql, vars) { + return executor.runUpdate(sql, vars); + }); + } + + /// Executes a custom insert statement and returns the last inserted rowid. + /// + /// You can tell moor which tables your query is going to affect by using the + /// [updates] parameter. Query-streams running on any of these tables will + /// then be re-run. + @protected + @visibleForTesting + Future customInsert(String query, + {List variables = const [], Set updates}) { + return _customWrite(query, variables, updates, (executor, sql, vars) { + return executor.runInsert(sql, vars); + }); + } + + /// Common logic for [customUpdate] and [customInsert] which takes care of + /// mapping the variables, running the query and optionally informing the + /// stream-queries. + Future _customWrite(String query, List variables, + Set updates, _CustomWriter writer) async { final engine = _resolvedEngine; final executor = engine.executor; final ctx = GenerationContext.fromDb(engine); final mappedArgs = variables.map((v) => v.mapToSimpleValue(ctx)).toList(); - final affectedRows = await executor - .doWhenOpened((_) => executor.runUpdate(query, mappedArgs)); + final result = await writer(executor, query, mappedArgs); if (updates != null) { await engine.streamQueries.handleTableUpdates(updates); } - return affectedRows; + return result; } /// Executes a custom select statement once. To use the variables, mark them diff --git a/moor/test/custom_queries_test.dart b/moor/test/custom_queries_test.dart index 4c63e77c..2d6d84ac 100644 --- a/moor/test/custom_queries_test.dart +++ b/moor/test/custom_queries_test.dart @@ -1,3 +1,4 @@ +import 'package:moor/moor.dart'; import 'package:test_api/test_api.dart'; import 'data/tables/todos.dart'; @@ -6,10 +7,12 @@ import 'data/utils/mocks.dart'; void main() { TodoDb db; MockExecutor executor; + MockStreamQueries streamQueries; setUp(() { executor = MockExecutor(); - db = TodoDb(executor); + streamQueries = MockStreamQueries(); + db = TodoDb(executor)..streamQueries = streamQueries; }); group('compiled custom queries', () { @@ -25,4 +28,23 @@ void main() { ); }); }); + + test('custom update informs stream queries', () async { + await db.customUpdate('UPDATE tbl SET a = ?', + variables: [Variable.withString('hi')], updates: {db.users}); + + verify(executor.runUpdate('UPDATE tbl SET a = ?', ['hi'])); + verify(streamQueries.handleTableUpdates({db.users})); + }); + + test('custom insert', () async { + when(executor.runInsert(any, any)).thenAnswer((_) => Future.value(32)); + + final id = + await db.customInsert('fake insert', variables: [Variable.withInt(3)]); + expect(id, 32); + + // shouldn't call stream queries - we didn't set the updates parameter + verifyNever(streamQueries.handleTableUpdates(any)); + }); } From 241baed0c3b7b1cda71ddcbaaad7cbe423ac059d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 29 Aug 2019 16:27:02 +0200 Subject: [PATCH 035/117] Generate query implementations for insert statements --- moor/test/data/tables/custom_tables.dart | 5 ++++- moor/test/data/tables/custom_tables.g.dart | 15 +++++++++++++++ moor_generator/lib/src/model/sql_query.dart | 4 +++- .../src/parser/sql/affected_tables_visitor.dart | 6 ++++++ .../lib/src/parser/sql/query_handler.dart | 9 +++++++-- moor_generator/lib/src/writer/query_writer.dart | 4 +++- sqlparser/lib/src/reader/parser/expressions.dart | 7 +++++-- sqlparser/lib/src/reader/parser/parser.dart | 8 +++++++- sqlparser/lib/src/reader/tokenizer/token.dart | 16 +++++++++++++++- 9 files changed, 65 insertions(+), 9 deletions(-) diff --git a/moor/test/data/tables/custom_tables.dart b/moor/test/data/tables/custom_tables.dart index 027b1735..71e9feb5 100644 --- a/moor/test/data/tables/custom_tables.dart +++ b/moor/test/data/tables/custom_tables.dart @@ -2,7 +2,10 @@ import 'package:moor/moor.dart'; part 'custom_tables.g.dart'; -@UseMoor(include: {'tables.moor'}) +@UseMoor( + include: {'tables.moor'}, + queries: {'writeConfig': 'REPLACE INTO config VALUES (:key, :value)'}, +) class CustomTablesDb extends _$CustomTablesDb { CustomTablesDb(QueryExecutor e) : super(e); diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 7a7b75c9..c5d6c701 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -812,6 +812,21 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Config get config => _config ??= Config(this); Mytable _mytable; Mytable get mytable => _mytable ??= Mytable(this); + Future writeConfig( + String key, + String value, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customInsert( + 'REPLACE INTO config VALUES (:key, :value)', + variables: [ + Variable.withString(key), + Variable.withString(value), + ], + updates: {config}, + ); + } + @override List get allTables => [noIds, withDefaults, withConstraints, config, mytable]; diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index f5b19cc5..855a934f 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -51,9 +51,11 @@ class SqlSelectQuery extends SqlQuery { class UpdatingQuery extends SqlQuery { final List updates; + final bool isInsert; UpdatingQuery(String name, AnalysisContext fromContext, - List variables, this.updates) + List variables, this.updates, + {this.isInsert = false}) : super(name, fromContext, variables); } diff --git a/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart b/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart index 795e6882..39679a66 100644 --- a/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart +++ b/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart @@ -52,4 +52,10 @@ class UpdatedTablesVisitor extends RecursiveVisitor { _addIfResolved(e.table); visitChildren(e); } + + @override + void visitInsertStatement(InsertStatement e) { + _addIfResolved(e.table); + visitChildren(e); + } } diff --git a/moor_generator/lib/src/parser/sql/query_handler.dart b/moor_generator/lib/src/parser/sql/query_handler.dart index f93d3e6f..bcea44b3 100644 --- a/moor_generator/lib/src/parser/sql/query_handler.dart +++ b/moor_generator/lib/src/parser/sql/query_handler.dart @@ -26,7 +26,9 @@ class QueryHandler { if (root is SelectStatement) { return _handleSelect(); - } else if (root is UpdateStatement || root is DeleteStatement) { + } else if (root is UpdateStatement || + root is DeleteStatement || + root is InsertStatement) { return _handleUpdate(); } else { throw StateError( @@ -39,8 +41,11 @@ class QueryHandler { context.root.accept(updatedFinder); _foundTables = updatedFinder.foundTables; + final isInsert = context.root is InsertStatement; + return UpdatingQuery(name, context, _foundVariables, - _foundTables.map(mapper.tableToMoor).toList()); + _foundTables.map(mapper.tableToMoor).toList(), + isInsert: isInsert); } SqlSelectQuery _handleSelect() { diff --git a/moor_generator/lib/src/writer/query_writer.dart b/moor_generator/lib/src/writer/query_writer.dart index 8c98d325..bf40cd6d 100644 --- a/moor_generator/lib/src/writer/query_writer.dart +++ b/moor_generator/lib/src/writer/query_writer.dart @@ -155,6 +155,8 @@ class QueryWriter { return customUpdate('', variables: [], updates: {}); } */ + final implName = _update.isInsert ? 'customInsert' : 'customUpdate'; + buffer.write('Future ${query.name}('); _writeParameters(buffer); buffer.write(') {\n'); @@ -162,7 +164,7 @@ class QueryWriter { _writeExpandedDeclarations(buffer); buffer ..write('return (operateOn ?? this).') - ..write('customUpdate(${_queryCode()},'); + ..write('$implName(${_queryCode()},'); _writeVariables(buffer); buffer.write(','); diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index 594f0a6e..bdf6879c 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -326,8 +326,10 @@ mixin ExpressionParser on ParserBase { break; case TokenType.colon: final colon = token; - final identifier = _consume(TokenType.identifier, - 'Expected an identifier for the named variable') as IdentifierToken; + final identifier = _consumeIdentifier( + 'Expected an identifier for the named variable', + lenient: true); + final content = identifier.identifier; return ColonNamedVariable(':$content')..setSpan(colon, identifier); default: @@ -392,6 +394,7 @@ mixin ExpressionParser on ParserBase { )..setSpan(name, _previous); } + @override TupleExpression _consumeTuple() { final firstToken = _consume(TokenType.leftParen, 'Expected opening parenthesis for tuple'); diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 05a39cbd..a47fbc7f 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -123,7 +123,13 @@ abstract class ParserBase { _error(message); } - IdentifierToken _consumeIdentifier(String message) { + /// Consumes an identifier. If [lenient] is true and the next token is not + /// an identifier but rather a [KeywordToken], that token will be converted + /// to an identifier. + IdentifierToken _consumeIdentifier(String message, {bool lenient = false}) { + if (lenient && _peek is KeywordToken) { + return (_advance() as KeywordToken).convertToIdentifier(); + } return _consume(TokenType.identifier, message) as IdentifierToken; } diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index ebc12054..65f30733 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -268,6 +268,11 @@ class IdentifierToken extends Token { /// Whether this identifier was escaped by putting it in "double ticks". final bool escaped; + /// Whether this identifier token is synthetic. We sometimes convert + /// [KeywordToken]s to identifiers if they're unambiguous, in which case + /// [synthetic] will be true on this token because it was not scanned as such. + final bool synthetic; + String get identifier { if (escaped) { return lexeme.substring(1, lexeme.length - 1); @@ -276,7 +281,7 @@ class IdentifierToken extends Token { } } - const IdentifierToken(this.escaped, FileSpan span) + const IdentifierToken(this.escaped, FileSpan span, {this.synthetic = false}) : super(TokenType.identifier, span); } @@ -295,7 +300,16 @@ class InlineDartToken extends Token { /// additional properties to ease syntax highlighting, as it allows us to find /// the keywords easily. class KeywordToken extends Token { + /// Whether this token has been used as an identifier while parsing. + bool isIdentifier; + KeywordToken(TokenType type, FileSpan span) : super(type, span); + + IdentifierToken convertToIdentifier() { + isIdentifier = true; + + return IdentifierToken(false, span, synthetic: false); + } } class TokenizerError { From 3cb00a4b31341015d3be957006e278e2e0c4f26a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 29 Aug 2019 16:33:15 +0200 Subject: [PATCH 036/117] Remove unnecessary "this." in insert companion constructor --- moor/example/example.g.dart | 14 +++++++------- moor/test/data/tables/custom_tables.g.dart | 6 +++--- moor/test/data/tables/todos.g.dart | 18 +++++++++--------- .../src/writer/update_companion_writer.dart | 5 +++-- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index 31dd4ec8..7b81c753 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -275,8 +275,8 @@ class RecipesCompanion extends UpdateCompanion { @required String title, @required String instructions, this.category = const Value.absent(), - }) : this.title = Value(title), - this.instructions = Value(instructions); + }) : title = Value(title), + instructions = Value(instructions); RecipesCompanion copyWith( {Value id, Value title, @@ -497,8 +497,8 @@ class IngredientsCompanion extends UpdateCompanion { this.id = const Value.absent(), @required String name, @required int caloriesPer100g, - }) : this.name = Value(name), - this.caloriesPer100g = Value(caloriesPer100g); + }) : name = Value(name), + caloriesPer100g = Value(caloriesPer100g); IngredientsCompanion copyWith( {Value id, Value name, Value caloriesPer100g}) { return IngredientsCompanion( @@ -709,9 +709,9 @@ class IngredientInRecipesCompanion extends UpdateCompanion { @required int recipe, @required int ingredient, @required int amountInGrams, - }) : this.recipe = Value(recipe), - this.ingredient = Value(ingredient), - this.amountInGrams = Value(amountInGrams); + }) : recipe = Value(recipe), + ingredient = Value(ingredient), + amountInGrams = Value(amountInGrams); IngredientInRecipesCompanion copyWith( {Value recipe, Value ingredient, Value amountInGrams}) { return IngredientInRecipesCompanion( diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index c5d6c701..d460692b 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -65,7 +65,7 @@ class NoIdsCompanion extends UpdateCompanion { }); NoIdsCompanion.insert({ @required Uint8List payload, - }) : this.payload = Value(payload); + }) : payload = Value(payload); NoIdsCompanion copyWith({Value payload}) { return NoIdsCompanion( payload: payload ?? this.payload, @@ -370,7 +370,7 @@ class WithConstraintsCompanion extends UpdateCompanion { this.a = const Value.absent(), @required int b, this.c = const Value.absent(), - }) : this.b = Value(b); + }) : b = Value(b); WithConstraintsCompanion copyWith( {Value a, Value b, Value c}) { return WithConstraintsCompanion( @@ -550,7 +550,7 @@ class ConfigCompanion extends UpdateCompanion { ConfigCompanion.insert({ @required String configKey, this.configValue = const Value.absent(), - }) : this.configKey = Value(configKey); + }) : configKey = Value(configKey); ConfigCompanion copyWith( {Value configKey, Value configValue}) { return ConfigCompanion( diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 99f3a799..74b0e7e2 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -139,7 +139,7 @@ class TodosTableCompanion extends UpdateCompanion { @required String content, this.targetDate = const Value.absent(), this.category = const Value.absent(), - }) : this.content = Value(content); + }) : content = Value(content); TodosTableCompanion copyWith( {Value id, Value title, @@ -368,7 +368,7 @@ class CategoriesCompanion extends UpdateCompanion { CategoriesCompanion.insert({ this.id = const Value.absent(), @required String description, - }) : this.description = Value(description); + }) : description = Value(description); CategoriesCompanion copyWith({Value id, Value description}) { return CategoriesCompanion( id: id ?? this.id, @@ -586,8 +586,8 @@ class UsersCompanion extends UpdateCompanion { this.isAwesome = const Value.absent(), @required Uint8List profilePicture, this.creationTime = const Value.absent(), - }) : this.name = Value(name), - this.profilePicture = Value(profilePicture); + }) : name = Value(name), + profilePicture = Value(profilePicture); UsersCompanion copyWith( {Value id, Value name, @@ -814,8 +814,8 @@ class SharedTodosCompanion extends UpdateCompanion { SharedTodosCompanion.insert({ @required int todo, @required int user, - }) : this.todo = Value(todo), - this.user = Value(user); + }) : todo = Value(todo), + user = Value(user); SharedTodosCompanion copyWith({Value todo, Value user}) { return SharedTodosCompanion( todo: todo ?? this.todo, @@ -1006,9 +1006,9 @@ class TableWithoutPKCompanion extends UpdateCompanion { @required int notReallyAnId, @required double someFloat, @required MyCustomObject custom, - }) : this.notReallyAnId = Value(notReallyAnId), - this.someFloat = Value(someFloat), - this.custom = Value(custom); + }) : notReallyAnId = Value(notReallyAnId), + someFloat = Value(someFloat), + custom = Value(custom); TableWithoutPKCompanion copyWith( {Value notReallyAnId, Value someFloat, diff --git a/moor_generator/lib/src/writer/update_companion_writer.dart b/moor_generator/lib/src/writer/update_companion_writer.dart index 9ed7f89b..8f65f73c 100644 --- a/moor_generator/lib/src/writer/update_companion_writer.dart +++ b/moor_generator/lib/src/writer/update_companion_writer.dart @@ -50,7 +50,8 @@ class UpdateCompanionWriter { // .insert({ // @required String a, // this.b = const Value.absent(), - // @required String b}): this.a = Value(a), this.b = Value(b); + // @required String b}): a = Value(a), b = Value(b); + // We don't need to use this. for the initializers, Dart figures that out. for (var column in table.columns) { final param = column.dartGetterName; @@ -75,7 +76,7 @@ class UpdateCompanionWriter { } final param = required.dartGetterName; - buffer.write('this.$param = Value($param)'); + buffer.write('$param = Value($param)'); } buffer.write(';\n'); From 876db0671e74eb90595015d9bb65201d49100e5f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 29 Aug 2019 21:09:20 +0200 Subject: [PATCH 037/117] Provide lints on insert statements that will fail --- moor_generator/lib/src/model/sql_query.dart | 2 + .../lib/src/parser/sql/lints/linter.dart | 82 +++++++++++++++++++ .../lib/src/parser/sql/query_handler.dart | 12 ++- .../lib/src/parser/sql/sql_parser.dart | 10 +++ sqlparser/lib/src/analysis/error.dart | 1 + 5 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 moor_generator/lib/src/parser/sql/lints/linter.dart diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 855a934f..75ad0f8e 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -10,6 +10,8 @@ final _leadingDigits = RegExp(r'^\d*'); abstract class SqlQuery { final String name; final AnalysisContext fromContext; + List lints; + String get sql => fromContext.sql; /// The variables that appear in the [sql] query. We support three kinds of diff --git a/moor_generator/lib/src/parser/sql/lints/linter.dart b/moor_generator/lib/src/parser/sql/lints/linter.dart new file mode 100644 index 00000000..69f2dccf --- /dev/null +++ b/moor_generator/lib/src/parser/sql/lints/linter.dart @@ -0,0 +1,82 @@ +import 'package:sqlparser/sqlparser.dart'; + +import '../query_handler.dart'; + +class Linter { + final QueryHandler handler; + final List lints = []; + + Linter(this.handler); + + void reportLints() { + handler.context.root.accept(_LintingVisitor(this)); + } +} + +class _LintingVisitor extends RecursiveVisitor { + final Linter linter; + + _LintingVisitor(this.linter); + + @override + void visitInsertStatement(InsertStatement e) { + final targeted = e.resolvedTargetColumns; + if (targeted == null) return; + + // First, check that the amount of values matches the declaration. + e.source.when( + isValues: (values) { + for (var tuple in values.values) { + if (tuple.expressions.length != targeted.length) { + linter.lints.add(AnalysisError( + type: AnalysisErrorType.other, + message: 'Expected tuple to have ${targeted.length} values', + relevantNode: tuple, + )); + } + } + }, + isSelect: (select) { + final columns = select.stmt.resolvedColumns; + + if (columns.length != targeted.length) { + linter.lints.add(AnalysisError( + type: AnalysisErrorType.other, + message: 'This select statement should return ${targeted.length} ' + 'columns, but actually returns ${columns.length}', + relevantNode: select.stmt, + )); + } + }, + ); + + // second, check that no required columns are left out + final specifiedTable = + linter.handler.mapper.tableToMoor(e.table.resolved as Table); + final required = + specifiedTable.columns.where((c) => c.requiredDuringInsert).toList(); + + if (required.isNotEmpty && e.source is DefaultValues) { + linter.lints.add(AnalysisError( + type: AnalysisErrorType.other, + message: 'This table has columns without default values, so defaults ' + 'can\'t be used for insert.', + relevantNode: e.table, + )); + } else { + final notPresent = required.where((c) => !targeted + .any((t) => t.name.toUpperCase() == c.name.name.toUpperCase())); + + if (notPresent.isNotEmpty) { + final msg = notPresent.join(', '); + + linter.lints.add(AnalysisError( + type: AnalysisErrorType.other, + message: 'Some columns are required but not present here. Expected ' + 'values for $msg.', + relevantNode: e.source.childNodes.first, + )); + } + } + } +} diff --git a/moor_generator/lib/src/parser/sql/query_handler.dart b/moor_generator/lib/src/parser/sql/query_handler.dart index bcea44b3..3bc25bea 100644 --- a/moor_generator/lib/src/parser/sql/query_handler.dart +++ b/moor_generator/lib/src/parser/sql/query_handler.dart @@ -5,6 +5,7 @@ import 'package:moor_generator/src/utils/type_converter_hint.dart'; import 'package:sqlparser/sqlparser.dart' hide ResultColumn; import 'affected_tables_visitor.dart'; +import 'lints/linter.dart'; class QueryHandler { final String name; @@ -19,11 +20,20 @@ class QueryHandler { QueryHandler(this.name, this.context, this.mapper); SqlQuery handle() { - final root = context.root; _foundVariables = mapper.extractVariables(context); _verifyNoSkippedIndexes(); + final query = _mapToMoor(); + final linter = Linter(this); + linter.reportLints(); + query.lints = linter.lints; + + return query; + } + + SqlQuery _mapToMoor() { + final root = context.root; if (root is SelectStatement) { return _handleSelect(); } else if (root is UpdateStatement || diff --git a/moor_generator/lib/src/parser/sql/sql_parser.dart b/moor_generator/lib/src/parser/sql/sql_parser.dart index a5c45c43..0f0ca885 100644 --- a/moor_generator/lib/src/parser/sql/sql_parser.dart +++ b/moor_generator/lib/src/parser/sql/sql_parser.dart @@ -54,5 +54,15 @@ class SqlParser { log.warning('Error while generating APIs for ${context.sql}', e, s); } }); + + // report lints + for (var query in foundQueries) { + for (var lint in query.lints) { + session.errors.add(MoorError( + critical: false, + message: 'Lint for ${query.name}: $lint', + )); + } + } } } diff --git a/sqlparser/lib/src/analysis/error.dart b/sqlparser/lib/src/analysis/error.dart index 5dd805b1..a7289f5c 100644 --- a/sqlparser/lib/src/analysis/error.dart +++ b/sqlparser/lib/src/analysis/error.dart @@ -56,4 +56,5 @@ enum AnalysisErrorType { ambiguousReference, unknownFunction, + other, } From 0ef56d61638bca1f721483a5ec7d55652f776660 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 30 Aug 2019 09:16:55 +0200 Subject: [PATCH 038/117] Use doWhenOpened for custom updates and inserts --- moor/lib/src/runtime/database.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moor/lib/src/runtime/database.dart b/moor/lib/src/runtime/database.dart index 4427d40b..0abeb098 100644 --- a/moor/lib/src/runtime/database.dart +++ b/moor/lib/src/runtime/database.dart @@ -207,7 +207,8 @@ mixin QueryEngine on DatabaseConnectionUser { final ctx = GenerationContext.fromDb(engine); final mappedArgs = variables.map((v) => v.mapToSimpleValue(ctx)).toList(); - final result = await writer(executor, query, mappedArgs); + final result = + await executor.doWhenOpened((e) => writer(e, query, mappedArgs)); if (updates != null) { await engine.streamQueries.handleTableUpdates(updates); From 4af370f0cbfb603879e414eb645ca2e22955dbe6 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 30 Aug 2019 19:32:44 +0200 Subject: [PATCH 039/117] Tests: Verify that an executor is open when used. --- moor/test/data/utils/mocks.dart | 31 +++++++++++++++++---- sqlparser/lib/src/reader/parser/schema.dart | 6 ++-- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/moor/test/data/utils/mocks.dart b/moor/test/data/utils/mocks.dart index b38bdec6..23566853 100644 --- a/moor/test/data/utils/mocks.dart +++ b/moor/test/data/utils/mocks.dart @@ -10,19 +10,40 @@ typedef Future _EnsureOpenAction(QueryExecutor e); class MockExecutor extends Mock implements QueryExecutor { final MockTransactionExecutor transactions = MockTransactionExecutor(); + var _opened = false; MockExecutor() { - when(runSelect(any, any)).thenAnswer((_) => Future.value([])); - when(runUpdate(any, any)).thenAnswer((_) => Future.value(0)); - when(runDelete(any, any)).thenAnswer((_) => Future.value(0)); - when(runInsert(any, any)).thenAnswer((_) => Future.value(0)); + when(runSelect(any, any)).thenAnswer((_) { + assert(_opened); + return Future.value([]); + }); + when(runUpdate(any, any)).thenAnswer((_) { + assert(_opened); + return Future.value(0); + }); + when(runDelete(any, any)).thenAnswer((_) { + assert(_opened); + return Future.value(0); + }); + when(runInsert(any, any)).thenAnswer((_) { + assert(_opened); + return Future.value(0); + }); + when(beginTransaction()).thenAnswer((_) { + assert(_opened); + return transactions; + }); + when(doWhenOpened(any)).thenAnswer((i) { + _opened = true; final action = i.positionalArguments.single as _EnsureOpenAction; return action(this); }); - when(beginTransaction()).thenAnswer((_) => transactions); + when(close()).thenAnswer((_) async { + _opened = false; + }); } } diff --git a/sqlparser/lib/src/reader/parser/schema.dart b/sqlparser/lib/src/reader/parser/schema.dart index 04634318..fc4c7551 100644 --- a/sqlparser/lib/src/reader/parser/schema.dart +++ b/sqlparser/lib/src/reader/parser/schema.dart @@ -15,7 +15,8 @@ mixin SchemaParser on ParserBase { ifNotExists = true; } - final tableIdentifier = _consumeIdentifier('Expected a table name'); + final tableIdentifier = + _consumeIdentifier('Expected a table name', lenient: true); // we don't currently support CREATE TABLE x AS SELECT ... statements _consume( @@ -213,7 +214,8 @@ mixin SchemaParser on ParserBase { String _constraintNameOrNull() { if (_matchOne(TokenType.constraint)) { - final name = _consumeIdentifier('Expect a name for the constraint here'); + final name = _consumeIdentifier('Expect a name for the constraint here', + lenient: true); return name.identifier; } return null; From fcdbc309b7aa2686c5815eca96cec9b5c1956456 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 30 Aug 2019 19:36:23 +0200 Subject: [PATCH 040/117] Fix resolver not finding window expressions --- sqlparser/lib/src/analysis/steps/reference_resolver.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlparser/lib/src/analysis/steps/reference_resolver.dart b/sqlparser/lib/src/analysis/steps/reference_resolver.dart index 046a18e9..28bb5043 100644 --- a/sqlparser/lib/src/analysis/steps/reference_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/reference_resolver.dart @@ -69,7 +69,7 @@ class ReferenceResolver extends RecursiveVisitor { @override void visitAggregateExpression(AggregateExpression e) { - if (e.windowName != null && e.resolved != null) { + if (e.windowName != null && e.resolved == null) { final resolved = e.scope.resolve(e.windowName); e.resolved = resolved; } From 23fca6196113eb0a3db8a12e4fbbb6892454c459 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 30 Aug 2019 23:09:22 +0200 Subject: [PATCH 041/117] Start with some refactoring in the generator The idea is that we have a setup that let's us use multiple backends (build, analyzer plugin, standalone) with maximum code sharing. --- moor_generator/.gitignore | 1 - moor_generator/lib/moor_generator.dart | 17 +----- .../lib/src/analyzer/dart/column_parser.dart | 45 ++++++++++++++ .../lib/src/analyzer/dart/parser.dart | 40 +++++++++++++ moor_generator/lib/src/analyzer/errors.dart | 44 ++++++++++++++ moor_generator/lib/src/analyzer/inputs.dart | 23 ++++++++ moor_generator/lib/src/analyzer/results.dart | 27 +++++++++ moor_generator/lib/src/analyzer/session.dart | 57 ++++++++++++++++++ moor_generator/lib/src/backends/backend.dart | 19 ++++++ .../lib/src/backends/build/moor_builder.dart | 21 +++++++ .../test/analyzer/dart/dart_test.dart | 47 +++++++++++++++ moor_generator/test/utils/test_backend.dart | 59 +++++++++++++++++++ 12 files changed, 384 insertions(+), 16 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/dart/column_parser.dart create mode 100644 moor_generator/lib/src/analyzer/dart/parser.dart create mode 100644 moor_generator/lib/src/analyzer/errors.dart create mode 100644 moor_generator/lib/src/analyzer/inputs.dart create mode 100644 moor_generator/lib/src/analyzer/results.dart create mode 100644 moor_generator/lib/src/analyzer/session.dart create mode 100644 moor_generator/lib/src/backends/backend.dart create mode 100644 moor_generator/lib/src/backends/build/moor_builder.dart create mode 100644 moor_generator/test/analyzer/dart/dart_test.dart create mode 100644 moor_generator/test/utils/test_backend.dart diff --git a/moor_generator/.gitignore b/moor_generator/.gitignore index fac28d89..b5033877 100644 --- a/moor_generator/.gitignore +++ b/moor_generator/.gitignore @@ -8,7 +8,6 @@ # Files and directories created by pub .dart_tool/ .packages -build/ # If you're building an application, you may want to check-in your pubspec.lock pubspec.lock diff --git a/moor_generator/lib/moor_generator.dart b/moor_generator/lib/moor_generator.dart index 0acf77d4..e90d6c14 100644 --- a/moor_generator/lib/moor_generator.dart +++ b/moor_generator/lib/moor_generator.dart @@ -1,17 +1,4 @@ import 'package:build/build.dart'; -import 'package:moor_generator/src/dao_generator.dart'; -import 'package:moor_generator/src/state/options.dart'; -import 'package:source_gen/source_gen.dart'; -import 'package:moor_generator/src/moor_generator.dart'; +import 'package:moor_generator/src/backends/build/moor_builder.dart'; -Builder moorBuilder(BuilderOptions options) { - final parsedOptions = MoorOptions.fromBuilder(options.config); - - return SharedPartBuilder( - [ - MoorGenerator(parsedOptions), - DaoGenerator(parsedOptions), - ], - 'moor', - ); -} +Builder moorBuilder(BuilderOptions options) => MoorBuilder(options); diff --git a/moor_generator/lib/src/analyzer/dart/column_parser.dart b/moor_generator/lib/src/analyzer/dart/column_parser.dart new file mode 100644 index 00000000..3f4370dc --- /dev/null +++ b/moor_generator/lib/src/analyzer/dart/column_parser.dart @@ -0,0 +1,45 @@ +part of 'parser.dart'; + +const String startInt = 'integer'; +const String startString = 'text'; +const String startBool = 'boolean'; +const String startDateTime = 'dateTime'; +const String startBlob = 'blob'; +const String startReal = 'real'; + +const Set starters = { + startInt, + startString, + startBool, + startDateTime, + startBlob, + startReal, +}; + +const String _methodNamed = 'named'; +const String _methodReferences = 'references'; +const String _methodAutoIncrement = 'autoIncrement'; +const String _methodWithLength = 'withLength'; +const String _methodNullable = 'nullable'; +const String _methodCustomConstraint = 'customConstraint'; +const String _methodDefault = 'withDefault'; +const String _methodMap = 'map'; + +/// Parses a single column defined in a Dart table. These columns are a chain +/// or [MethodInvocation]s. An example getter might look like this: +/// ```dart +/// IntColumn get id => integer().autoIncrement()(); +/// ``` +/// The last call `()` is a [FunctionExpressionInvocation], the entries for +/// before that (in this case `autoIncrement()` and `integer()` are a) +/// [MethodInvocation]. We work our way through that syntax until we hit a +/// method that starts the chain (contained in [starters]). By visiting all +/// the invocations on our way, we can extract the constraint for the column +/// (e.g. its name, whether it has auto increment, is a primary key and so on). +class ColumnParser { + final MoorDartParser base; + + ColumnParser(this.base); + + SpecifiedColumn parse(MethodDeclaration getter, MethodElement element) {} +} diff --git a/moor_generator/lib/src/analyzer/dart/parser.dart b/moor_generator/lib/src/analyzer/dart/parser.dart new file mode 100644 index 00000000..3f9b2ae7 --- /dev/null +++ b/moor_generator/lib/src/analyzer/dart/parser.dart @@ -0,0 +1,40 @@ +import 'package:analyzer/dart/analysis/results.dart'; +import 'package:meta/meta.dart'; +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/element/element.dart'; +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/model/specified_column.dart'; + +part 'column_parser.dart'; + +class MoorDartParser { + final DartTask task; + + MoorDartParser(this.task); + + @visibleForTesting + Expression returnExpressionOfMethod(MethodDeclaration method) { + final body = method.body; + + if (!(body is ExpressionFunctionBody)) { + task.reportError(ErrorInDartCode( + affectedElement: method.declaredElement, + severity: Severity.criticalError, + message: + 'This method must have an expression body (user => instead of {return ...})', + )); + return null; + } + + return (method.body as ExpressionFunctionBody).expression; + } + + Future loadElementDeclaration( + Element element) async { + final resolvedLibrary = await element.library.session + .getResolvedLibraryByElement(element.library); + + return resolvedLibrary.getElementDeclaration(element); + } +} diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart new file mode 100644 index 00000000..85ef0b0c --- /dev/null +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -0,0 +1,44 @@ +import 'package:analyzer/dart/element/element.dart'; +import 'package:collection/collection.dart'; + +/// Base class for errors that can be presented to an user. +class MoorError { + final Severity severity; + + MoorError(this.severity); +} + +class ErrorInDartCode extends MoorError { + final String message; + final Element affectedElement; + + ErrorInDartCode( + {this.message, + this.affectedElement, + Severity severity = Severity.warning}) + : super(severity); +} + +class ErrorSink { + final List _errors = []; + UnmodifiableListView get errors => UnmodifiableListView(_errors); + + void report(MoorError error) { + _errors.add(error); + } +} + +enum Severity { + /// A severe error. We might not be able to generate correct or consistent + /// code when errors with these severity are present. + criticalError, + + /// An error. The generated code won't have major problems, but might cause + /// runtime errors. For instance, this is used when we get sql that has + /// semantic errors. + error, + + warning, + info, + hint +} diff --git a/moor_generator/lib/src/analyzer/inputs.dart b/moor_generator/lib/src/analyzer/inputs.dart new file mode 100644 index 00000000..6ef9bac7 --- /dev/null +++ b/moor_generator/lib/src/analyzer/inputs.dart @@ -0,0 +1,23 @@ +import 'package:analyzer/dart/element/element.dart'; + +/// Inputs coming from an external system (such as the analyzer, the build +/// package, etc.) that will be further analyzed by moor. +abstract class Input { + final String path; + + Input(this.path); +} + +/// Input for Dart files that have already been analyzed. +class DartInput extends Input { + final LibraryElement library; + + DartInput(String path, this.library) : super(path); +} + +/// Input for a `.moor` file +class MoorInput extends Input { + final String content; + + MoorInput(String path, this.content) : super(path); +} diff --git a/moor_generator/lib/src/analyzer/results.dart b/moor_generator/lib/src/analyzer/results.dart new file mode 100644 index 00000000..6b839f72 --- /dev/null +++ b/moor_generator/lib/src/analyzer/results.dart @@ -0,0 +1,27 @@ +import 'package:meta/meta.dart'; +import 'package:analyzer/dart/element/element.dart'; +import 'package:moor_generator/src/model/specified_dao.dart'; +import 'package:moor_generator/src/model/specified_database.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; + +abstract class ParsedFile {} + +class ParsedDartFile extends ParsedFile { + final LibraryElement library; + + final List declaredTables; + final List declaredDaos; + final List declaredDatabases; + + ParsedDartFile( + {@required this.library, + this.declaredTables = const [], + this.declaredDaos = const [], + this.declaredDatabases = const []}); +} + +class ParsedMoorFile extends ParsedFile { + final List declaredTables; + + ParsedMoorFile(this.declaredTables); +} diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart new file mode 100644 index 00000000..23cfd185 --- /dev/null +++ b/moor_generator/lib/src/analyzer/session.dart @@ -0,0 +1,57 @@ +import 'dart:async'; + +import 'package:analyzer/dart/element/element.dart'; +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/results.dart'; +import 'package:moor_generator/src/backends/backend.dart'; + +/// Will store cached data about files that have already been analyzed. +class MoorSession { + MoorSession(); + + Future startDartTask(BackendTask backendTask) async { + final library = await backendTask.resolveDart(backendTask.entrypoint); + return DartTask(this, backendTask, library); + } +} + +/// Used to parse and analyze a single file. +abstract class FileTask { + final BackendTask backendTask; + final MoorSession session; + + final ErrorSink errors = ErrorSink(); + + FileTask(this.backendTask, this.session); + + void reportError(MoorError error) => errors.report(error); + + FutureOr compute(); +} + +/// Session used to parse a Dart file and extract table information. +class DartTask extends FileTask { + final LibraryElement library; + + DartTask(MoorSession session, BackendTask task, this.library) + : super(task, session); + + @override + FutureOr compute() { + // TODO: implement compute + return null; + } +} + +class MoorTask extends FileTask { + final List content; + + MoorTask(BackendTask task, MoorSession session, this.content) + : super(task, session); + + @override + FutureOr compute() { + // TODO: implement compute + return null; + } +} diff --git a/moor_generator/lib/src/backends/backend.dart b/moor_generator/lib/src/backends/backend.dart new file mode 100644 index 00000000..5ab51df8 --- /dev/null +++ b/moor_generator/lib/src/backends/backend.dart @@ -0,0 +1,19 @@ +import 'package:analyzer/dart/element/element.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; + +/// A backend for the moor generator. +/// +/// Currently, we only have a backend based on the build package, but we can +/// extend this to a backend for an analyzer plugin or a standalone tool. +abstract class Backend { + final MoorSession session = MoorSession(); +} + +/// Used to analyze a single file via ([entrypoint]). The other methods can be +/// used to read imports used by the other files. +abstract class BackendTask { + String get entrypoint; + + Future resolveDart(String path); + Future readMoor(String path); +} diff --git a/moor_generator/lib/src/backends/build/moor_builder.dart b/moor_generator/lib/src/backends/build/moor_builder.dart new file mode 100644 index 00000000..e88276f5 --- /dev/null +++ b/moor_generator/lib/src/backends/build/moor_builder.dart @@ -0,0 +1,21 @@ +import 'package:build/build.dart'; +import 'package:moor_generator/src/dao_generator.dart'; +import 'package:moor_generator/src/moor_generator.dart'; +import 'package:moor_generator/src/state/options.dart'; +import 'package:source_gen/source_gen.dart'; + +class MoorBuilder extends SharedPartBuilder { + factory MoorBuilder(BuilderOptions options) { + final parsedOptions = MoorOptions.fromBuilder(options.config); + + final generators = [ + MoorGenerator(parsedOptions), + DaoGenerator(parsedOptions), + ]; + + return MoorBuilder._(generators, 'moor'); + } + + MoorBuilder._(List generators, String name) + : super(generators, name); +} diff --git a/moor_generator/test/analyzer/dart/dart_test.dart b/moor_generator/test/analyzer/dart/dart_test.dart new file mode 100644 index 00000000..e77a47cd --- /dev/null +++ b/moor_generator/test/analyzer/dart/dart_test.dart @@ -0,0 +1,47 @@ +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/element/element.dart'; +import 'package:moor_generator/src/analyzer/dart/parser.dart'; +import 'package:test/test.dart'; + +import '../../utils/test_backend.dart'; + +void main() { + test('return expression of methods', () async { + final backend = TestBackend({ + 'test_lib|main.dart': r''' + class Test { + String get getter => 'foo'; + String function() => 'bar'; + String invalid() { + return 'baz'; + } + } + ''' + }); + + final backendTask = backend.startTask('test_lib|main.dart'); + final dartTask = await backend.session.startDartTask(backendTask); + final parser = MoorDartParser(dartTask); + + Future _loadDeclaration(Element element) async { + final declaration = await parser.loadElementDeclaration(element); + return declaration.node as MethodDeclaration; + } + + void _verifyReturnExpressionMatches(Element element, String source) async { + final node = await _loadDeclaration(element); + expect(parser.returnExpressionOfMethod(node).toSource(), source); + } + + final testClass = dartTask.library.getType('Test'); + + _verifyReturnExpressionMatches(testClass.getGetter('getter'), "'foo'"); + _verifyReturnExpressionMatches(testClass.getMethod('function'), "'bar'"); + + final invalidDecl = await _loadDeclaration(testClass.getMethod('invalid')); + expect(parser.returnExpressionOfMethod(invalidDecl), isNull); + expect(dartTask.errors.errors, isNotEmpty); + + backend.finish(); + }); +} diff --git a/moor_generator/test/utils/test_backend.dart b/moor_generator/test/utils/test_backend.dart new file mode 100644 index 00000000..a08eebb9 --- /dev/null +++ b/moor_generator/test/utils/test_backend.dart @@ -0,0 +1,59 @@ +import 'dart:async'; + +import 'package:analyzer/dart/element/element.dart'; +import 'package:build/build.dart'; +import 'package:build_test/build_test.dart'; +import 'package:moor_generator/src/backends/backend.dart'; + +class TestBackend extends Backend { + final Map fakeContent; + Resolver _resolver; + + final Completer _initCompleter = Completer(); + final Completer _finish = Completer(); + + /// Future that completes when this backend is ready, which happens when all + /// input files have been parsed and analyzed by the Dart analyzer. + Future get _ready => _initCompleter.future; + + TestBackend(this.fakeContent) { + _init(); + } + + void _init() { + resolveSources(fakeContent, (r) { + _resolver = r; + _initCompleter.complete(); + return _finish.future; + }); + } + + BackendTask startTask(String path) { + return _TestBackendTask(this, path); + } + + void finish() { + _finish.complete(); + } +} + +class _TestBackendTask extends BackendTask { + final TestBackend backend; + + @override + final String entrypoint; + + _TestBackendTask(this.backend, this.entrypoint); + + @override + Future readMoor(String path) async { + await backend._ready; + return backend.fakeContent[path]; + } + + @override + Future resolveDart(String path) async { + await backend._ready; + return await backend._resolver.libraryFor(AssetId.parse(path)); + } +} From 6e89a319aded6df68402408137cd158d8ebd4b48 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 31 Aug 2019 12:22:25 +0200 Subject: [PATCH 042/117] Refactored parser can now parse Dart tables --- .../lib/src/analyzer/dart/column_parser.dart | 173 +++++++++++++++++- .../lib/src/analyzer/dart/parser.dart | 58 +++++- .../lib/src/analyzer/dart/table_parser.dart | 122 ++++++++++++ moor_generator/lib/src/analyzer/errors.dart | 2 + .../test/analyzer/dart/table_parser_test.dart | 138 ++++++++++++++ 5 files changed, 491 insertions(+), 2 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/dart/table_parser.dart create mode 100644 moor_generator/test/analyzer/dart/table_parser_test.dart diff --git a/moor_generator/lib/src/analyzer/dart/column_parser.dart b/moor_generator/lib/src/analyzer/dart/column_parser.dart index 3f4370dc..97ef1e9e 100644 --- a/moor_generator/lib/src/analyzer/dart/column_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/column_parser.dart @@ -25,6 +25,10 @@ const String _methodCustomConstraint = 'customConstraint'; const String _methodDefault = 'withDefault'; const String _methodMap = 'map'; +const String _errorMessage = 'This getter does not create a valid column that ' + 'can be parsed by moor. Please refer to the readme from moor to see how ' + 'columns are formed. If you have any questions, feel free to raise an issue.'; + /// Parses a single column defined in a Dart table. These columns are a chain /// or [MethodInvocation]s. An example getter might look like this: /// ```dart @@ -41,5 +45,172 @@ class ColumnParser { ColumnParser(this.base); - SpecifiedColumn parse(MethodDeclaration getter, MethodElement element) {} + SpecifiedColumn parse(MethodDeclaration getter, Element element) { + final expr = base.returnExpressionOfMethod(getter); + + if (!(expr is FunctionExpressionInvocation)) { + base.task.reportError(ErrorInDartCode( + affectedElement: getter.declaredElement, + message: _errorMessage, + severity: Severity.criticalError, + )); + return null; + } + + var remainingExpr = + (expr as FunctionExpressionInvocation).function as MethodInvocation; + + String foundStartMethod; + String foundExplicitName; + String foundCustomConstraint; + Expression foundDefaultExpression; + Expression createdTypeConverter; + DartType typeConverterRuntime; + var nullable = false; + + final foundFeatures = []; + + while (true) { + final methodName = remainingExpr.methodName.name; + + if (starters.contains(methodName)) { + foundStartMethod = methodName; + break; + } + + switch (methodName) { + case _methodNamed: + if (foundExplicitName != null) { + base.task.reportError( + ErrorInDartCode( + severity: Severity.warning, + affectedElement: getter.declaredElement, + message: + "You're setting more than one name here, the first will " + 'be used', + ), + ); + } + + foundExplicitName = base.readStringLiteral( + remainingExpr.argumentList.arguments.first, () { + base.task.reportError( + ErrorInDartCode( + severity: Severity.error, + affectedElement: getter.declaredElement, + message: + 'This table name is cannot be resolved! Please only use ' + 'a constant string as parameter for .named().', + ), + ); + }); + break; + case _methodReferences: + break; + case _methodWithLength: + final args = remainingExpr.argumentList; + final minArg = base.findNamedArgument(args, 'min'); + final maxArg = base.findNamedArgument(args, 'max'); + + foundFeatures.add(LimitingTextLength.withLength( + min: base.readIntLiteral(minArg, () {}), + max: base.readIntLiteral(maxArg, () {}), + )); + break; + case _methodAutoIncrement: + foundFeatures.add(AutoIncrement()); + // a column declared as auto increment is always a primary key + foundFeatures.add(const PrimaryKey()); + break; + case _methodNullable: + nullable = true; + break; + case _methodCustomConstraint: + foundCustomConstraint = base.readStringLiteral( + remainingExpr.argumentList.arguments.first, () { + base.task.reportError( + ErrorInDartCode( + severity: Severity.warning, + affectedElement: getter.declaredElement, + message: + 'This constraint is cannot be resolved! Please only use ' + 'a constant string as parameter for .customConstraint().', + ), + ); + }); + break; + case _methodDefault: + final args = remainingExpr.argumentList; + final expression = args.arguments.single; + foundDefaultExpression = expression; + break; + case _methodMap: + final args = remainingExpr.argumentList; + final expression = args.arguments.single; + + // the map method has a parameter type that resolved to the runtime + // type of the custom object + final type = remainingExpr.typeArgumentTypes.single; + + createdTypeConverter = expression; + typeConverterRuntime = type; + break; + } + + // We're not at a starting method yet, so we need to go deeper! + final inner = (remainingExpr.target) as MethodInvocation; + remainingExpr = inner; + } + + ColumnName name; + if (foundExplicitName != null) { + name = ColumnName.explicitly(foundExplicitName); + } else { + name = ColumnName.implicitly(ReCase(getter.name.name).snakeCase); + } + + final columnType = _startMethodToColumnType(foundStartMethod); + + UsedTypeConverter converter; + if (createdTypeConverter != null && typeConverterRuntime != null) { + converter = UsedTypeConverter( + expression: createdTypeConverter, + mappedType: typeConverterRuntime, + sqlType: columnType); + } + + return SpecifiedColumn( + type: columnType, + dartGetterName: getter.name.name, + name: name, + overriddenJsonName: _readJsonKey(element), + customConstraints: foundCustomConstraint, + nullable: nullable, + features: foundFeatures, + defaultArgument: foundDefaultExpression?.toSource(), + typeConverter: converter); + } + + ColumnType _startMethodToColumnType(String startMethod) { + return const { + startBool: ColumnType.boolean, + startString: ColumnType.text, + startInt: ColumnType.integer, + startDateTime: ColumnType.datetime, + startBlob: ColumnType.blob, + startReal: ColumnType.real, + }[startMethod]; + } + + String _readJsonKey(Element getter) { + final annotations = getter.metadata; + final object = annotations.singleWhere((e) { + final value = e.computeConstantValue(); + return isFromMoor(value.type) && value.type.name == 'JsonKey'; + }, orElse: () => null); + + if (object == null) return null; + + return object.constantValue.getField('key').toStringValue(); + } } diff --git a/moor_generator/lib/src/analyzer/dart/parser.dart b/moor_generator/lib/src/analyzer/dart/parser.dart index 3f9b2ae7..fb21528d 100644 --- a/moor_generator/lib/src/analyzer/dart/parser.dart +++ b/moor_generator/lib/src/analyzer/dart/parser.dart @@ -1,17 +1,40 @@ import 'package:analyzer/dart/analysis/results.dart'; +import 'package:analyzer/dart/element/type.dart'; import 'package:meta/meta.dart'; import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; +import 'package:moor/sqlite_keywords.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/model/specified_column.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/used_type_converter.dart'; +import 'package:moor_generator/src/utils/names.dart'; +import 'package:moor_generator/src/utils/type_utils.dart'; +import 'package:recase/recase.dart'; part 'column_parser.dart'; +part 'table_parser.dart'; class MoorDartParser { final DartTask task; - MoorDartParser(this.task); + ColumnParser _columnParser; + TableParser _tableParser; + + MoorDartParser(this.task) { + _columnParser = ColumnParser(this); + _tableParser = TableParser(this); + } + + Future parseTable(ClassElement classElement) { + return _tableParser.parseTable(classElement); + } + + Future parseColumn( + MethodDeclaration declaration, Element element) { + return Future.value(_columnParser.parse(declaration, element)); + } @visibleForTesting Expression returnExpressionOfMethod(MethodDeclaration method) { @@ -37,4 +60,37 @@ class MoorDartParser { return resolvedLibrary.getElementDeclaration(element); } + + String readStringLiteral(Expression expression, void onError()) { + if (!(expression is StringLiteral)) { + onError(); + } else { + final value = (expression as StringLiteral).stringValue; + if (value == null) { + onError(); + } else { + return value; + } + } + + return null; + } + + int readIntLiteral(Expression expression, void onError()) { + if (!(expression is IntegerLiteral)) { + onError(); + // ignore: avoid_returning_null + return null; + } else { + return (expression as IntegerLiteral).value; + } + } + + Expression findNamedArgument(ArgumentList args, String argName) { + final argument = args.arguments.singleWhere( + (e) => e is NamedExpression && e.name.label.name == argName, + orElse: () => null) as NamedExpression; + + return argument?.expression; + } } diff --git a/moor_generator/lib/src/analyzer/dart/table_parser.dart b/moor_generator/lib/src/analyzer/dart/table_parser.dart new file mode 100644 index 00000000..6f214a27 --- /dev/null +++ b/moor_generator/lib/src/analyzer/dart/table_parser.dart @@ -0,0 +1,122 @@ +part of 'parser.dart'; + +/// Parses a [SpecifiedTable] from a Dart class. +class TableParser { + final MoorDartParser base; + + TableParser(this.base); + + Future parseTable(ClassElement element) async { + final sqlName = await _parseTableName(element); + if (sqlName == null) return null; + + final columns = await _parseColumns(element); + + final table = SpecifiedTable( + fromClass: element, + columns: columns, + sqlName: escapeIfNeeded(sqlName), + dartTypeName: _readDartTypeName(element), + primaryKey: await _readPrimaryKey(element, columns), + ); + + var index = 0; + for (var converter in table.converters) { + converter + ..index = index++ + ..table = table; + } + + return table; + } + + String _readDartTypeName(ClassElement element) { + final nameAnnotation = element.metadata.singleWhere( + (e) => e.computeConstantValue().type.name == 'DataClassName', + orElse: () => null); + + if (nameAnnotation == null) { + return dataClassNameForClassName(element.name); + } else { + return nameAnnotation.constantValue.getField('name').toStringValue(); + } + } + + Future _parseTableName(ClassElement element) async { + // todo allow override via a field (final String tableName = '') as well + + final tableNameGetter = element.getGetter('tableName'); + if (tableNameGetter == null) { + // class does not override tableName. So just use the dart class name + // instead. Will use placed_orders for a class called PlacedOrders + return ReCase(element.name).snakeCase; + } + + // we expect something like get tableName => "myTableName", the getter + // must do nothing more complicated + final tableNameDeclaration = + await base.loadElementDeclaration(tableNameGetter); + final returnExpr = base.returnExpressionOfMethod( + tableNameDeclaration.node as MethodDeclaration); + + final tableName = base.readStringLiteral(returnExpr, () { + base.task.reportError(ErrorInDartCode( + severity: Severity.criticalError, + message: + 'This getter must return a string literal, and do nothing more', + affectedElement: tableNameGetter)); + }); + + return tableName; + } + + Future> _readPrimaryKey( + ClassElement element, List columns) async { + final primaryKeyGetter = element.getGetter('primaryKey'); + if (primaryKeyGetter == null) { + return null; + } + + final resolved = await base.loadElementDeclaration(primaryKeyGetter); + final ast = resolved.node as MethodDeclaration; + final body = ast.body; + if (body is! ExpressionFunctionBody) { + base.task.reportError(ErrorInDartCode( + affectedElement: primaryKeyGetter, + message: 'This must return a set literal using the => syntax!')); + return null; + } + final expression = (body as ExpressionFunctionBody).expression; + final parsedPrimaryKey = {}; + + if (expression is SetOrMapLiteral) { + for (var entry in expression.elements) { + if (entry is Identifier) { + final column = columns + .singleWhere((column) => column.dartGetterName == entry.name); + parsedPrimaryKey.add(column); + } else { + print('Unexpected entry in expression.elements: $entry'); + } + } + } else { + base.task.reportError(ErrorInDartCode( + affectedElement: primaryKeyGetter, + message: 'This must return a set literal!')); + } + + return parsedPrimaryKey; + } + + Future> _parseColumns(ClassElement element) { + final columns = element.fields + .where((field) => isColumn(field.type) && field.getter != null); + + return Future.wait(columns.map((field) async { + final resolved = await base.loadElementDeclaration(field.getter); + final node = resolved.node as MethodDeclaration; + + return await base.parseColumn(node, field.getter); + })); + } +} diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index 85ef0b0c..9c599117 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -38,6 +38,8 @@ enum Severity { /// semantic errors. error, + /// A warning is used when the code affected is technically valid, but + /// unlikely to do what the user expects. warning, info, hint diff --git a/moor_generator/test/analyzer/dart/table_parser_test.dart b/moor_generator/test/analyzer/dart/table_parser_test.dart new file mode 100644 index 00000000..bc6fa258 --- /dev/null +++ b/moor_generator/test/analyzer/dart/table_parser_test.dart @@ -0,0 +1,138 @@ +import 'package:moor_generator/src/analyzer/dart/parser.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/model/specified_column.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:test/test.dart'; + +import '../../utils/test_backend.dart'; + +void main() { + TestBackend backend; + DartTask dartTask; + MoorDartParser parser; + setUpAll(() { + backend = TestBackend({ + 'test_lib|main.dart': r''' + import 'package:moor/moor.dart'; + + class TableWithCustomName extends Table { + @override String get tableName => 'my-fancy-table'; + } + + class Users extends Table { + IntColumn get id => integer().autoIncrement()(); + TextColumn get name => text().named("user_name").withLength(min: 6, max: 32)(); + TextColumn get onlyMax => text().withLength(max: 100)(); + + DateTimeColumn get defaults => dateTime().withDefault(currentDate)(); + } + + class CustomPrimaryKey extends Table { + IntColumn get partA => integer()(); + IntColumn get partB => integer().customConstraint('custom')(); + + @override + Set get primaryKey => {partA, partB}; + } + + class WrongName extends Table { + String constructTableName() => 'my-table-name'; + String get tableName => constructTableName(); + } + ''' + }); + }); + tearDownAll(() { + backend.finish(); + }); + + setUp(() async { + final task = backend.startTask('test_lib|main.dart'); + dartTask = await backend.session.startDartTask(task); + parser = MoorDartParser(dartTask); + }); + + Future parse(String name) async { + return parser.parseTable(dartTask.library.getType(name)); + } + + group('table names', () { + test('use overridden name', () async { + final parsed = await parse('TableWithCustomName'); + expect(parsed.sqlName, equals('my-fancy-table')); + }); + + test('use re-cased class name', () async { + final parsed = await parse('Users'); + expect(parsed.sqlName, equals('users')); + }); + + test('should not parse for complex methods', () async { + await parse('WrongName'); + expect(dartTask.errors.errors, isNotEmpty); + }); + }); + + group('Columns', () { + test('should use field name if no name has been set explicitly', () async { + final table = await parse('Users'); + final idColumn = + table.columns.singleWhere((col) => col.name.name == 'id'); + + expect(idColumn.name, equals(ColumnName.implicitly('id'))); + }); + + test('should use explicit name, if it exists', () async { + final table = await parse('Users'); + final idColumn = + table.columns.singleWhere((col) => col.name.name == 'user_name'); + + expect(idColumn.name, equals(ColumnName.explicitly('user_name'))); + }); + + test('should parse min and max length for text columns', () async { + final table = await parse('Users'); + final idColumn = + table.columns.singleWhere((col) => col.name.name == 'user_name'); + + expect(idColumn.features, + contains(LimitingTextLength.withLength(min: 6, max: 32))); + }); + + test('should only parse max length when relevant', () async { + final table = await parse('Users'); + final idColumn = + table.columns.singleWhere((col) => col.dartGetterName == 'onlyMax'); + + expect( + idColumn.features, contains(LimitingTextLength.withLength(max: 100))); + }); + + test('parses custom constraints', () async { + final table = await parse('CustomPrimaryKey'); + + final partA = + table.columns.singleWhere((c) => c.dartGetterName == 'partA'); + final partB = + table.columns.singleWhere((c) => c.dartGetterName == 'partB'); + + expect(partB.customConstraints, 'custom'); + expect(partA.customConstraints, isNull); + }); + + test('parsed default values', () async { + final table = await parse('Users'); + final defaultsColumn = + table.columns.singleWhere((c) => c.name.name == 'defaults'); + + expect(defaultsColumn.defaultArgument.toString(), 'currentDate'); + }); + }); + + test('parses custom primary keys', () async { + final table = await parse('CustomPrimaryKey'); + + expect(table.primaryKey, containsAll(table.columns)); + expect(table.columns.any((column) => column.hasAI), isFalse); + }); +} From ad8bdba4b83bf8b789f3b71ade7c574037939df5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 2 Sep 2019 17:26:30 +0200 Subject: [PATCH 043/117] Start migration of writer code --- .../lib/src/analyzer/dart/parser.dart | 5 + .../dart}/use_dao_parser.dart | 15 +- .../dart}/use_moor_parser.dart | 17 +- moor_generator/lib/src/analyzer/errors.dart | 20 +- .../moor/create_table_reader.dart} | 29 +-- .../lib/src/analyzer/moor/parser.dart | 45 ++++ moor_generator/lib/src/analyzer/session.dart | 93 +++++++- .../sql_queries}/affected_tables_visitor.dart | 0 .../sql_queries}/query_handler.dart | 5 +- .../sql_queries}/sql_parser.dart | 19 +- .../sql_queries}/type_mapping.dart | 0 .../lib/src/backends/build/build_backend.dart | 28 +++ .../lib/src/backends/build/moor_builder.dart | 6 +- .../{state => backends/build}/options.dart | 2 + .../lib/src/parser/column_parser.dart | 223 ------------------ .../lib/src/parser/moor/moor_analyzer.dart | 63 ----- moor_generator/lib/src/parser/parser.dart | 64 ----- .../lib/src/parser/table_parser.dart | 129 ---------- moor_generator/lib/src/state/errors.dart | 17 -- .../lib/src/state/generator_state.dart | 39 --- moor_generator/lib/src/state/session.dart | 129 ---------- .../lib/src/writer/database_writer.dart | 12 +- .../writer/{ => queries}/query_writer.dart | 167 +++++++------ .../{ => queries}/result_set_writer.dart | 1 + .../{ => tables}/data_class_writer.dart | 10 +- .../src/writer/{ => tables}/table_writer.dart | 2 +- .../{ => tables}/update_companion_writer.dart | 0 .../memoized_getter.dart} | 0 .../lib/src/{state => writer}/writer.dart | 48 +++- 29 files changed, 364 insertions(+), 824 deletions(-) rename moor_generator/lib/src/{parser => analyzer/dart}/use_dao_parser.dart (59%) rename moor_generator/lib/src/{parser => analyzer/dart}/use_moor_parser.dart (63%) rename moor_generator/lib/src/{parser/moor/parsed_moor_file.dart => analyzer/moor/create_table_reader.dart} (82%) create mode 100644 moor_generator/lib/src/analyzer/moor/parser.dart rename moor_generator/lib/src/{parser/sql => analyzer/sql_queries}/affected_tables_visitor.dart (100%) rename moor_generator/lib/src/{parser/sql => analyzer/sql_queries}/query_handler.dart (95%) rename moor_generator/lib/src/{parser/sql => analyzer/sql_queries}/sql_parser.dart (72%) rename moor_generator/lib/src/{parser/sql => analyzer/sql_queries}/type_mapping.dart (100%) create mode 100644 moor_generator/lib/src/backends/build/build_backend.dart rename moor_generator/lib/src/{state => backends/build}/options.dart (96%) delete mode 100644 moor_generator/lib/src/parser/column_parser.dart delete mode 100644 moor_generator/lib/src/parser/moor/moor_analyzer.dart delete mode 100644 moor_generator/lib/src/parser/parser.dart delete mode 100644 moor_generator/lib/src/parser/table_parser.dart delete mode 100644 moor_generator/lib/src/state/errors.dart delete mode 100644 moor_generator/lib/src/state/generator_state.dart delete mode 100644 moor_generator/lib/src/state/session.dart rename moor_generator/lib/src/writer/{ => queries}/query_writer.dart (68%) rename moor_generator/lib/src/writer/{ => queries}/result_set_writer.dart (93%) rename moor_generator/lib/src/writer/{ => tables}/data_class_writer.dart (97%) rename moor_generator/lib/src/writer/{ => tables}/table_writer.dart (99%) rename moor_generator/lib/src/writer/{ => tables}/update_companion_writer.dart (100%) rename moor_generator/lib/src/writer/{utils.dart => utils/memoized_getter.dart} (100%) rename moor_generator/lib/src/{state => writer}/writer.dart (62%) diff --git a/moor_generator/lib/src/analyzer/dart/parser.dart b/moor_generator/lib/src/analyzer/dart/parser.dart index fb21528d..c6aa77e6 100644 --- a/moor_generator/lib/src/analyzer/dart/parser.dart +++ b/moor_generator/lib/src/analyzer/dart/parser.dart @@ -7,14 +7,19 @@ import 'package:moor/sqlite_keywords.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/model/specified_column.dart'; +import 'package:moor_generator/src/model/specified_dao.dart'; +import 'package:moor_generator/src/model/specified_database.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/used_type_converter.dart'; import 'package:moor_generator/src/utils/names.dart'; import 'package:moor_generator/src/utils/type_utils.dart'; import 'package:recase/recase.dart'; +import 'package:source_gen/source_gen.dart'; part 'column_parser.dart'; part 'table_parser.dart'; +part 'use_dao_parser.dart'; +part 'use_moor_parser.dart'; class MoorDartParser { final DartTask task; diff --git a/moor_generator/lib/src/parser/use_dao_parser.dart b/moor_generator/lib/src/analyzer/dart/use_dao_parser.dart similarity index 59% rename from moor_generator/lib/src/parser/use_dao_parser.dart rename to moor_generator/lib/src/analyzer/dart/use_dao_parser.dart index 603d5366..796cee79 100644 --- a/moor_generator/lib/src/parser/use_dao_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/use_dao_parser.dart @@ -1,12 +1,9 @@ -import 'package:analyzer/dart/element/element.dart'; -import 'package:moor_generator/src/model/specified_dao.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:source_gen/source_gen.dart'; +part of 'parser.dart'; class UseDaoParser { - final GeneratorSession session; + final DartTask dartTask; - UseDaoParser(this.session); + UseDaoParser(this.dartTask); /// If [element] has a `@UseDao` annotation, parses the database model /// declared by that class and the referenced tables. @@ -24,11 +21,11 @@ class UseDaoParser { ?.map((e) => e.toStringValue()) ?? {}; - final parsedTables = await session.parseTables(tableTypes, element); - parsedTables.addAll(await session.resolveIncludes(includes)); + final parsedTables = await dartTask.parseTables(tableTypes, element); + parsedTables.addAll(await dartTask.resolveIncludes(includes)); final parsedQueries = - await session.parseQueries(queryStrings, parsedTables); + await dartTask.parseQueries(queryStrings, parsedTables); return SpecifiedDao(element, parsedTables, parsedQueries); } diff --git a/moor_generator/lib/src/parser/use_moor_parser.dart b/moor_generator/lib/src/analyzer/dart/use_moor_parser.dart similarity index 63% rename from moor_generator/lib/src/parser/use_moor_parser.dart rename to moor_generator/lib/src/analyzer/dart/use_moor_parser.dart index e15bde2f..80ec18f3 100644 --- a/moor_generator/lib/src/parser/use_moor_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/use_moor_parser.dart @@ -1,13 +1,9 @@ -import 'package:analyzer/dart/element/element.dart'; -import 'package:analyzer/dart/element/type.dart'; -import 'package:moor_generator/src/model/specified_database.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:source_gen/source_gen.dart'; +part of 'parser.dart'; class UseMoorParser { - final GeneratorSession session; + final DartTask task; - UseMoorParser(this.session); + UseMoorParser(this.task); /// If [element] has a `@UseMoor` annotation, parses the database model /// declared by that class and the referenced tables. @@ -25,11 +21,10 @@ class UseMoorParser { ?.map((e) => e.toStringValue()) ?? {}; - final parsedTables = await session.parseTables(tableTypes, element); - parsedTables.addAll(await session.resolveIncludes(includes)); + final parsedTables = await task.parseTables(tableTypes, element); + parsedTables.addAll(await task.resolveIncludes(includes)); - final parsedQueries = - await session.parseQueries(queryStrings, parsedTables); + final parsedQueries = await task.parseQueries(queryStrings, parsedTables); final daoTypes = _readDaoTypes(annotation); return SpecifiedDatabase(element, parsedTables, daoTypes, parsedQueries); diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index 9c599117..8673eaf6 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -1,22 +1,34 @@ import 'package:analyzer/dart/element/element.dart'; import 'package:collection/collection.dart'; +import 'package:meta/meta.dart'; +import 'package:source_span/source_span.dart'; /// Base class for errors that can be presented to an user. class MoorError { final Severity severity; + final String message; - MoorError(this.severity); + MoorError({@required this.severity, this.message}); } class ErrorInDartCode extends MoorError { - final String message; final Element affectedElement; ErrorInDartCode( - {this.message, + {String message, this.affectedElement, Severity severity = Severity.warning}) - : super(severity); + : super(severity: severity, message: message); +} + +class ErrorInMoorFile extends MoorError { + final FileSpan span; + + ErrorInMoorFile( + {@required this.span, + String message, + Severity severity = Severity.warning}) + : super(message: message, severity: severity); } class ErrorSink { diff --git a/moor_generator/lib/src/parser/moor/parsed_moor_file.dart b/moor_generator/lib/src/analyzer/moor/create_table_reader.dart similarity index 82% rename from moor_generator/lib/src/parser/moor/parsed_moor_file.dart rename to moor_generator/lib/src/analyzer/moor/create_table_reader.dart index aabdae85..21bf136d 100644 --- a/moor_generator/lib/src/parser/moor/parsed_moor_file.dart +++ b/moor_generator/lib/src/analyzer/moor/create_table_reader.dart @@ -1,40 +1,17 @@ +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/used_type_converter.dart'; -import 'package:moor_generator/src/parser/sql/type_mapping.dart'; import 'package:moor_generator/src/utils/names.dart'; import 'package:moor_generator/src/utils/string_escaper.dart'; import 'package:recase/recase.dart'; import 'package:sqlparser/sqlparser.dart'; -/* -We're in the process of defining what a .moor file could actually look like. -At the moment, we only support "CREATE TABLE" statements: -``` // content of a .moor file -CREATE TABLE users ( - id INTEGER NOT NULL PRIMARY KEY AUTO_INCREMENT, - name VARCHAR(100) NOT NULL, -) -``` - -In the future, we'd also like to support -- import statements between moor files -- import statements from moor files referencing tables declared via the Dart DSL -- declaring statements in these files, similar to how compiled statements work - with the annotation. - */ - -class ParsedMoorFile { - final List declaredTables; - - ParsedMoorFile(this.declaredTables); -} - -class CreateTable { +class CreateTableReader { /// The AST of this `CREATE TABLE` statement. final ParseResult ast; - CreateTable(this.ast); + CreateTableReader(this.ast); SpecifiedTable extractTable(TypeMapper mapper) { final table = diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart new file mode 100644 index 00000000..50d2d8f9 --- /dev/null +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -0,0 +1,45 @@ +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/moor/create_table_reader.dart'; +import 'package:moor_generator/src/analyzer/results.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:sqlparser/sqlparser.dart'; + +class MoorParser { + final MoorTask task; + + MoorParser(this.task); + + Future parseAndAnalyze() { + final results = + SqlEngine(useMoorExtensions: true).parseMultiple(task.content); + + final createdReaders = []; + + for (var parsedStmt in results) { + if (parsedStmt.rootNode is CreateTableStatement) { + createdReaders.add(CreateTableReader(parsedStmt)); + } else { + task.reportError(ErrorInMoorFile( + span: parsedStmt.rootNode.span, + message: 'At the moment, only CREATE TABLE statements are supported' + 'in .moor files')); + } + } + + // all results have the same list of errors + final sqlErrors = results.isEmpty ? [] : results.first.errors; + + for (var error in sqlErrors) { + task.reportError(ErrorInMoorFile( + span: error.token.span, + message: error.message, + )); + } + + final createdTables = + createdReaders.map((r) => r.extractTable(task.mapper)).toList(); + final parsedFile = ParsedMoorFile(createdTables); + + return Future.value(parsedFile); + } +} diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 23cfd185..bbb51cb7 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -1,18 +1,37 @@ import 'dart:async'; +import 'package:analyzer/dart/constant/value.dart'; import 'package:analyzer/dart/element/element.dart'; +import 'package:analyzer/dart/element/type.dart'; +import 'package:moor/moor.dart' show Table; +import 'package:moor_generator/src/analyzer/dart/parser.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/moor/parser.dart'; import 'package:moor_generator/src/analyzer/results.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/sql_parser.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; import 'package:moor_generator/src/backends/backend.dart'; +import 'package:moor_generator/src/model/specified_dao.dart'; +import 'package:moor_generator/src/model/specified_database.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:source_gen/source_gen.dart'; /// Will store cached data about files that have already been analyzed. class MoorSession { MoorSession(); - Future startDartTask(BackendTask backendTask) async { - final library = await backendTask.resolveDart(backendTask.entrypoint); + Future startDartTask(BackendTask backendTask, {String uri}) async { + final input = uri ?? backendTask.entrypoint; + final library = await backendTask.resolveDart(input); return DartTask(this, backendTask, library); } + + Future startMoorTask(BackendTask backendTask, {String uri}) async { + final input = uri ?? backendTask.entrypoint; + final source = await backendTask.readMoor(input); + return MoorTask(backendTask, this, source); + } } /// Used to parse and analyze a single file. @@ -31,27 +50,89 @@ abstract class FileTask { /// Session used to parse a Dart file and extract table information. class DartTask extends FileTask { + static const tableTypeChecker = const TypeChecker.fromRuntime(Table); + final LibraryElement library; + MoorDartParser _parser; + MoorDartParser get parser => _parser; DartTask(MoorSession session, BackendTask task, this.library) - : super(task, session); + : super(task, session) { + _parser = MoorDartParser(this); + } @override FutureOr compute() { // TODO: implement compute return null; } + + /// Parses a [SpecifiedDatabase] from the [ClassElement] which was annotated + /// with `@UseMoor` and the [annotation] reader that reads the `@UseMoor` + /// annotation. + Future parseDatabase( + ClassElement element, ConstantReader annotation) { + return UseMoorParser(this).parseDatabase(element, annotation); + } + + /// Parses a [SpecifiedDao] from a class declaration that has a `UseDao` + /// [annotation]. + Future parseDao( + ClassElement element, ConstantReader annotation) { + return UseDaoParser(this).parseDao(element, annotation); + } + + /// Resolves a [SpecifiedTable] for the class of each [DartType] in [types]. + /// The [initializedBy] element should be the piece of code that caused the + /// parsing (e.g. the database class that is annotated with `@UseMoor`). This + /// will allow for more descriptive error messages. + Future> parseTables( + Iterable types, Element initializedBy) { + return Future.wait(types.map((type) { + if (!tableTypeChecker.isAssignableFrom(type.element)) { + reportError(ErrorInDartCode( + severity: Severity.criticalError, + message: 'The type $type is not a moor table', + affectedElement: initializedBy, + )); + return null; + } else { + return parser.parseTable(type.element as ClassElement); + } + })).then((list) => List.from(list)); // make growable + } + + /// Reads all tables declared in sql by a `.moor` file in [paths]. + Future> resolveIncludes(Iterable paths) { + return Stream.fromFutures( + paths.map((path) => session.startMoorTask(backendTask, uri: path))) + .asyncMap((task) => task.compute()) + .expand((file) => file.declaredTables) + .toList(); + } + + Future> parseQueries( + Map fromAnnotation, + List availableTables) { + // no queries declared, so there is no point in starting a sql engine + if (fromAnnotation.isEmpty) return Future.value([]); + + final parser = SqlParser(this, availableTables, fromAnnotation)..parse(); + + return Future.value(parser.foundQueries); + } } class MoorTask extends FileTask { - final List content; + final String content; + final TypeMapper mapper = TypeMapper(); MoorTask(BackendTask task, MoorSession session, this.content) : super(task, session); @override FutureOr compute() { - // TODO: implement compute - return null; + final parser = MoorParser(this); + return parser.parseAndAnalyze(); } } diff --git a/moor_generator/lib/src/parser/sql/affected_tables_visitor.dart b/moor_generator/lib/src/analyzer/sql_queries/affected_tables_visitor.dart similarity index 100% rename from moor_generator/lib/src/parser/sql/affected_tables_visitor.dart rename to moor_generator/lib/src/analyzer/sql_queries/affected_tables_visitor.dart diff --git a/moor_generator/lib/src/parser/sql/query_handler.dart b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart similarity index 95% rename from moor_generator/lib/src/parser/sql/query_handler.dart rename to moor_generator/lib/src/analyzer/sql_queries/query_handler.dart index bcea44b3..5fcd87d4 100644 --- a/moor_generator/lib/src/parser/sql/query_handler.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart @@ -1,11 +1,14 @@ import 'package:moor_generator/src/model/sql_query.dart'; import 'package:moor_generator/src/model/used_type_converter.dart'; -import 'package:moor_generator/src/parser/sql/type_mapping.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; import 'package:moor_generator/src/utils/type_converter_hint.dart'; import 'package:sqlparser/sqlparser.dart' hide ResultColumn; import 'affected_tables_visitor.dart'; +/// Maps an [AnalysisContext] from the sqlparser to a [SqlQuery] from this +/// generator package by determining its type, return columns, variables and so +/// on. class QueryHandler { final String name; final AnalysisContext context; diff --git a/moor_generator/lib/src/parser/sql/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart similarity index 72% rename from moor_generator/lib/src/parser/sql/sql_parser.dart rename to moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index a5c45c43..2fc74859 100644 --- a/moor_generator/lib/src/parser/sql/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -1,16 +1,16 @@ import 'package:analyzer/dart/constant/value.dart'; import 'package:build/build.dart'; -import 'package:moor_generator/src/state/errors.dart'; +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; -import 'package:moor_generator/src/parser/sql/query_handler.dart'; -import 'package:moor_generator/src/parser/sql/type_mapping.dart'; -import 'package:moor_generator/src/state/session.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/query_handler.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; import 'package:sqlparser/sqlparser.dart' hide ResultColumn; class SqlParser { final List tables; - final GeneratorSession session; + final FileTask task; final Map definedQueries; final TypeMapper _mapper = TypeMapper(); @@ -18,7 +18,7 @@ class SqlParser { final List foundQueries = []; - SqlParser(this.session, this.tables, this.definedQueries); + SqlParser(this.task, this.tables, this.definedQueries); void _spawnEngine() { _engine = SqlEngine(); @@ -36,14 +36,15 @@ class SqlParser { try { context = _engine.analyze(sql); } catch (e, s) { - session.errors.add(MoorError( - critical: true, + task.reportError(MoorError( + severity: Severity.criticalError, message: 'Error while trying to parse $sql: $e, $s')); return; } for (var error in context.errors) { - session.errors.add(MoorError( + task.reportError(MoorError( + severity: Severity.warning, message: 'The sql query $sql is invalid: $error', )); } diff --git a/moor_generator/lib/src/parser/sql/type_mapping.dart b/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart similarity index 100% rename from moor_generator/lib/src/parser/sql/type_mapping.dart rename to moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart diff --git a/moor_generator/lib/src/backends/build/build_backend.dart b/moor_generator/lib/src/backends/build/build_backend.dart new file mode 100644 index 00000000..f511cd7c --- /dev/null +++ b/moor_generator/lib/src/backends/build/build_backend.dart @@ -0,0 +1,28 @@ +import 'package:analyzer/dart/element/element.dart'; +import 'package:build/build.dart'; +import 'package:moor_generator/src/backends/backend.dart'; + +class BuildBackend extends Backend {} + +class BuildBackendTask extends BackendTask { + final BuildStep step; + + BuildBackendTask(this.step); + + @override + String get entrypoint => step.inputId.path; + + AssetId _resolve(String uri) { + return AssetId.resolve(uri, from: step.inputId); + } + + @override + Future readMoor(String path) { + return step.readAsString(_resolve(path)); + } + + @override + Future resolveDart(String path) { + return step.resolver.libraryFor(_resolve(path)); + } +} diff --git a/moor_generator/lib/src/backends/build/moor_builder.dart b/moor_generator/lib/src/backends/build/moor_builder.dart index e88276f5..66698c07 100644 --- a/moor_generator/lib/src/backends/build/moor_builder.dart +++ b/moor_generator/lib/src/backends/build/moor_builder.dart @@ -1,10 +1,14 @@ import 'package:build/build.dart'; +import 'package:moor_generator/src/backends/build/build_backend.dart'; import 'package:moor_generator/src/dao_generator.dart'; import 'package:moor_generator/src/moor_generator.dart'; -import 'package:moor_generator/src/state/options.dart'; import 'package:source_gen/source_gen.dart'; +part 'options.dart'; + class MoorBuilder extends SharedPartBuilder { + final BuildBackend backend = BuildBackend(); + factory MoorBuilder(BuilderOptions options) { final parsedOptions = MoorOptions.fromBuilder(options.config); diff --git a/moor_generator/lib/src/state/options.dart b/moor_generator/lib/src/backends/build/options.dart similarity index 96% rename from moor_generator/lib/src/state/options.dart rename to moor_generator/lib/src/backends/build/options.dart index 9d4e6665..c3bdf26c 100644 --- a/moor_generator/lib/src/state/options.dart +++ b/moor_generator/lib/src/backends/build/options.dart @@ -1,3 +1,5 @@ +part of 'moor_builder.dart'; + class MoorOptions { final bool generateFromJsonStringConstructor; diff --git a/moor_generator/lib/src/parser/column_parser.dart b/moor_generator/lib/src/parser/column_parser.dart deleted file mode 100644 index a02ddc65..00000000 --- a/moor_generator/lib/src/parser/column_parser.dart +++ /dev/null @@ -1,223 +0,0 @@ -import 'package:analyzer/dart/ast/ast.dart'; -import 'package:analyzer/dart/element/element.dart'; -import 'package:analyzer/dart/element/type.dart'; -import 'package:moor_generator/src/model/used_type_converter.dart'; -import 'package:moor_generator/src/state/errors.dart'; -import 'package:moor_generator/src/model/specified_column.dart'; -import 'package:moor_generator/src/parser/parser.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:moor_generator/src/utils/type_utils.dart'; -import 'package:recase/recase.dart'; - -const String startInt = 'integer'; -const String startString = 'text'; -const String startBool = 'boolean'; -const String startDateTime = 'dateTime'; -const String startBlob = 'blob'; -const String startReal = 'real'; - -final Set starters = { - startInt, - startString, - startBool, - startDateTime, - startBlob, - startReal, -}; - -const String _methodNamed = 'named'; -const String _methodReferences = 'references'; -const String _methodAutoIncrement = 'autoIncrement'; -const String _methodWithLength = 'withLength'; -const String _methodNullable = 'nullable'; -const String _methodCustomConstraint = 'customConstraint'; -const String _methodDefault = 'withDefault'; -const String _methodMap = 'map'; - -const String _errorMessage = 'This getter does not create a valid column that ' - 'can be parsed by moor. Please refer to the readme from moor to see how ' - 'columns are formed. If you have any questions, feel free to raise an issue.'; - -class ColumnParser extends ParserBase { - ColumnParser(GeneratorSession session) : super(session); - - SpecifiedColumn parse(MethodDeclaration getter, Element getterElement) { - /* - These getters look like this: ... get id => integer().autoIncrement()(); - The last () is a FunctionExpressionInvocation, the entries before that - (here autoIncrement and integer) are MethodInvocations. - We go through each of the method invocations until we hit one that starts - the chain (integer, text, boolean, etc.). From each method in the chain, - we can extract what it means for the column (name, auto increment, PK, - constraints...). - */ - - final expr = returnExpressionOfMethod(getter); - - if (!(expr is FunctionExpressionInvocation)) { - session.errors.add(MoorError( - affectedElement: getter.declaredElement, - message: _errorMessage, - critical: true, - )); - - return null; - } - - var remainingExpr = - (expr as FunctionExpressionInvocation).function as MethodInvocation; - - String foundStartMethod; - String foundExplicitName; - String foundCustomConstraint; - Expression foundDefaultExpression; - Expression createdTypeConverter; - DartType typeConverterRuntime; - var nullable = false; - - final foundFeatures = []; - - while (true) { - final methodName = remainingExpr.methodName.name; - - if (starters.contains(methodName)) { - foundStartMethod = methodName; - break; - } - - switch (methodName) { - case _methodNamed: - if (foundExplicitName != null) { - session.errors.add( - MoorError( - critical: false, - affectedElement: getter.declaredElement, - message: - "You're setting more than one name here, the first will " - 'be used', - ), - ); - } - - foundExplicitName = - readStringLiteral(remainingExpr.argumentList.arguments.first, () { - session.errors.add( - MoorError( - critical: false, - affectedElement: getter.declaredElement, - message: - 'This table name is cannot be resolved! Please only use ' - 'a constant string as parameter for .named().', - ), - ); - }); - break; - case _methodReferences: - break; - case _methodWithLength: - final args = remainingExpr.argumentList; - final minArg = findNamedArgument(args, 'min'); - final maxArg = findNamedArgument(args, 'max'); - - foundFeatures.add(LimitingTextLength.withLength( - min: readIntLiteral(minArg, () {}), - max: readIntLiteral(maxArg, () {}), - )); - break; - case _methodAutoIncrement: - foundFeatures.add(AutoIncrement()); - // a column declared as auto increment is always a primary key - foundFeatures.add(const PrimaryKey()); - break; - case _methodNullable: - nullable = true; - break; - case _methodCustomConstraint: - foundCustomConstraint = - readStringLiteral(remainingExpr.argumentList.arguments.first, () { - session.errors.add( - MoorError( - critical: false, - affectedElement: getter.declaredElement, - message: - 'This constraint is cannot be resolved! Please only use ' - 'a constant string as parameter for .customConstraint().', - ), - ); - }); - break; - case _methodDefault: - final args = remainingExpr.argumentList; - final expression = args.arguments.single; - foundDefaultExpression = expression; - break; - case _methodMap: - final args = remainingExpr.argumentList; - final expression = args.arguments.single; - - // the map method has a parameter type that resolved to the runtime - // type of the custom object - final type = remainingExpr.typeArgumentTypes.single; - - createdTypeConverter = expression; - typeConverterRuntime = type; - break; - } - - // We're not at a starting method yet, so we need to go deeper! - final inner = (remainingExpr.target) as MethodInvocation; - remainingExpr = inner; - } - - ColumnName name; - if (foundExplicitName != null) { - name = ColumnName.explicitly(foundExplicitName); - } else { - name = ColumnName.implicitly(ReCase(getter.name.name).snakeCase); - } - - final columnType = _startMethodToColumnType(foundStartMethod); - - UsedTypeConverter converter; - if (createdTypeConverter != null && typeConverterRuntime != null) { - converter = UsedTypeConverter( - expression: createdTypeConverter, - mappedType: typeConverterRuntime, - sqlType: columnType); - } - - return SpecifiedColumn( - type: columnType, - dartGetterName: getter.name.name, - name: name, - overriddenJsonName: _readJsonKey(getterElement), - customConstraints: foundCustomConstraint, - nullable: nullable, - features: foundFeatures, - defaultArgument: foundDefaultExpression?.toSource(), - typeConverter: converter); - } - - ColumnType _startMethodToColumnType(String startMethod) { - return const { - startBool: ColumnType.boolean, - startString: ColumnType.text, - startInt: ColumnType.integer, - startDateTime: ColumnType.datetime, - startBlob: ColumnType.blob, - startReal: ColumnType.real, - }[startMethod]; - } - - String _readJsonKey(Element getter) { - final annotations = getter.metadata; - final object = annotations.singleWhere((e) { - final value = e.computeConstantValue(); - return isFromMoor(value.type) && value.type.name == 'JsonKey'; - }, orElse: () => null); - - if (object == null) return null; - - return object.constantValue.getField('key').toStringValue(); - } -} diff --git a/moor_generator/lib/src/parser/moor/moor_analyzer.dart b/moor_generator/lib/src/parser/moor/moor_analyzer.dart deleted file mode 100644 index 342c27b4..00000000 --- a/moor_generator/lib/src/parser/moor/moor_analyzer.dart +++ /dev/null @@ -1,63 +0,0 @@ -import 'package:moor_generator/src/parser/moor/parsed_moor_file.dart'; -import 'package:source_span/source_span.dart'; -import 'package:sqlparser/sqlparser.dart'; - -/// Parses and analyzes the experimental `.moor` files containing sql -/// statements. -class MoorAnalyzer { - /// Content of the `.moor` file we're analyzing. - final String content; - - MoorAnalyzer(this.content); - - Future analyze() { - final results = SqlEngine().parseMultiple(content); - - final createdTables = []; - final errors = []; - - for (var parsedStmt in results) { - if (parsedStmt.rootNode is CreateTableStatement) { - createdTables.add(CreateTable(parsedStmt)); - } else { - errors.add( - MoorParsingError( - parsedStmt.rootNode.span, - message: - 'At the moment, only CREATE TABLE statements are supported in .moor files', - ), - ); - } - } - - // all results have the same list of errors - final sqlErrors = results.isEmpty ? [] : results.first.errors; - - for (var error in sqlErrors) { - errors.add(MoorParsingError(error.token.span, message: error.message)); - } - - final parsedFile = ParsedMoorFile(createdTables); - - return Future.value(MoorParsingResult(parsedFile, errors)); - } -} - -class MoorParsingResult { - final ParsedMoorFile parsedFile; - final List errors; - - MoorParsingResult(this.parsedFile, this.errors); -} - -class MoorParsingError { - final FileSpan span; - final String message; - - MoorParsingError(this.span, {this.message}); - - @override - String toString() { - return span.message(message, color: true); - } -} diff --git a/moor_generator/lib/src/parser/parser.dart b/moor_generator/lib/src/parser/parser.dart deleted file mode 100644 index b2c112df..00000000 --- a/moor_generator/lib/src/parser/parser.dart +++ /dev/null @@ -1,64 +0,0 @@ -import 'package:analyzer/dart/ast/ast.dart'; -import 'package:moor_generator/src/state/errors.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/state/session.dart'; - -class Parser { - List specifiedTables; - - void init() async {} -} - -class ParserBase { - final GeneratorSession session; - - ParserBase(this.session); - - Expression returnExpressionOfMethod(MethodDeclaration method) { - final body = method.body; - - if (!(body is ExpressionFunctionBody)) { - session.errors.add(MoorError( - affectedElement: method.declaredElement, - critical: true, - message: - 'This method must have an expression body (use => instead of {return ...})')); - return null; - } - - return (method.body as ExpressionFunctionBody).expression; - } - - String readStringLiteral(Expression expression, void onError()) { - if (!(expression is StringLiteral)) { - onError(); - } else { - final value = (expression as StringLiteral).stringValue; - if (value == null) { - onError(); - } else { - return value; - } - } - - return null; - } - - int readIntLiteral(Expression expression, void onError()) { - if (!(expression is IntegerLiteral)) { - onError(); - // ignore: avoid_returning_null - return null; - } else { - return (expression as IntegerLiteral).value; - } - } - - Expression findNamedArgument(ArgumentList args, String argName) { - final argument = args.arguments.singleWhere( - (e) => e is NamedExpression && e.name.label.name == argName, - orElse: () => null) as NamedExpression; - - return argument?.expression; - } -} diff --git a/moor_generator/lib/src/parser/table_parser.dart b/moor_generator/lib/src/parser/table_parser.dart deleted file mode 100644 index 0f552c80..00000000 --- a/moor_generator/lib/src/parser/table_parser.dart +++ /dev/null @@ -1,129 +0,0 @@ -import 'package:analyzer/dart/ast/ast.dart'; -import 'package:analyzer/dart/element/element.dart'; -import 'package:moor_generator/src/state/errors.dart'; -import 'package:moor_generator/src/model/specified_column.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/parser/parser.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:moor_generator/src/utils/names.dart'; -import 'package:moor_generator/src/utils/type_utils.dart'; -import 'package:recase/recase.dart'; -import 'package:moor/sqlite_keywords.dart'; - -class TableParser extends ParserBase { - TableParser(GeneratorSession session) : super(session); - - Future parse(ClassElement element) async { - final sqlName = await _parseTableName(element); - if (sqlName == null) return null; - - final columns = await _parseColumns(element); - - final table = SpecifiedTable( - fromClass: element, - columns: columns, - sqlName: escapeIfNeeded(sqlName), - dartTypeName: _readDartTypeName(element), - primaryKey: await _readPrimaryKey(element, columns), - ); - - var index = 0; - for (var converter in table.converters) { - converter - ..index = index++ - ..table = table; - } - - return table; - } - - String _readDartTypeName(ClassElement element) { - final nameAnnotation = element.metadata.singleWhere( - (e) => e.computeConstantValue().type.name == 'DataClassName', - orElse: () => null); - - if (nameAnnotation == null) { - return dataClassNameForClassName(element.name); - } else { - return nameAnnotation.constantValue.getField('name').toStringValue(); - } - } - - Future _parseTableName(ClassElement element) async { - // todo allow override via a field (final String tableName = '') as well - - final tableNameGetter = element.getGetter('tableName'); - if (tableNameGetter == null) { - // class does not override tableName. So just use the dart class name - // instead. Will use placed_orders for a class called PlacedOrders - return ReCase(element.name).snakeCase; - } - - // we expect something like get tableName => "myTableName", the getter - // must do nothing more complicated - final tableNameDeclaration = - await session.loadElementDeclaration(tableNameGetter); - final returnExpr = returnExpressionOfMethod( - tableNameDeclaration.node as MethodDeclaration); - - final tableName = readStringLiteral(returnExpr, () { - session.errors.add(MoorError( - critical: true, - message: - 'This getter must return a string literal, and do nothing more', - affectedElement: tableNameGetter)); - }); - - return tableName; - } - - Future> _readPrimaryKey( - ClassElement element, List columns) async { - final primaryKeyGetter = element.getGetter('primaryKey'); - if (primaryKeyGetter == null) { - return null; - } - - final resolved = await session.loadElementDeclaration(primaryKeyGetter); - final ast = resolved.node as MethodDeclaration; - final body = ast.body; - if (body is! ExpressionFunctionBody) { - session.errors.add(MoorError( - affectedElement: primaryKeyGetter, - message: 'This must return a set literal using the => syntax!')); - return null; - } - final expression = (body as ExpressionFunctionBody).expression; - final parsedPrimaryKey = {}; - - if (expression is SetOrMapLiteral) { - for (var entry in expression.elements) { - if (entry is Identifier) { - final column = columns - .singleWhere((column) => column.dartGetterName == entry.name); - parsedPrimaryKey.add(column); - } else { - print('Unexpected entry in expression.elements: $entry'); - } - } - } else { - session.errors.add(MoorError( - affectedElement: primaryKeyGetter, - message: 'This must return a set literal!')); - } - - return parsedPrimaryKey; - } - - Future> _parseColumns(ClassElement element) { - final columns = element.fields - .where((field) => isColumn(field.type) && field.getter != null); - - return Future.wait(columns.map((field) async { - final resolved = await session.loadElementDeclaration(field.getter); - final node = resolved.node as MethodDeclaration; - - return await session.parseColumn(node, field.getter); - })); - } -} diff --git a/moor_generator/lib/src/state/errors.dart b/moor_generator/lib/src/state/errors.dart deleted file mode 100644 index 81b47928..00000000 --- a/moor_generator/lib/src/state/errors.dart +++ /dev/null @@ -1,17 +0,0 @@ -import 'package:analyzer/dart/element/element.dart'; - -class MoorError { - final bool critical; - final String message; - final Element affectedElement; - - MoorError({this.critical = false, this.message, this.affectedElement}); -} - -class ErrorStore { - final List errors = []; - - void add(MoorError error) => errors.add(error); - - bool get hasCriticalError => errors.any((e) => e.critical); -} diff --git a/moor_generator/lib/src/state/generator_state.dart b/moor_generator/lib/src/state/generator_state.dart deleted file mode 100644 index 40c651e5..00000000 --- a/moor_generator/lib/src/state/generator_state.dart +++ /dev/null @@ -1,39 +0,0 @@ -import 'package:analyzer/dart/element/type.dart'; -import 'package:build/build.dart'; -import 'package:moor/moor.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:source_gen/source_gen.dart'; - -import 'options.dart'; - -GeneratorState _state; - -/// Uses the created instance of the generator state or creates one via the -/// [create] callback if necessary. -GeneratorState useState(GeneratorState Function() create) { - return _state ??= create(); -} - -class GeneratorState { - final MoorOptions options; - - final Map> _foundTables = {}; - final tableTypeChecker = const TypeChecker.fromRuntime(Table); - - GeneratorState(this.options); - - GeneratorSession startSession(BuildStep step) { - return GeneratorSession(this, step); - } - - /// Parses the [SpecifiedTable] from a [type]. As this operation is very - /// expensive, we always try to only perform it once. - /// - /// The [resolve] function is responsible for performing the actual analysis - /// and it will be called when the [type] has not yet been resolved. - Future parseTable( - DartType type, Future Function() resolve) { - return _foundTables.putIfAbsent(type, resolve); - } -} diff --git a/moor_generator/lib/src/state/session.dart b/moor_generator/lib/src/state/session.dart deleted file mode 100644 index 6bb65486..00000000 --- a/moor_generator/lib/src/state/session.dart +++ /dev/null @@ -1,129 +0,0 @@ -import 'package:analyzer/dart/analysis/results.dart'; -import 'package:analyzer/dart/ast/ast.dart'; -import 'package:analyzer/dart/constant/value.dart'; -import 'package:analyzer/dart/element/element.dart'; -import 'package:analyzer/dart/element/type.dart'; -import 'package:build/build.dart'; -import 'package:moor_generator/src/model/specified_column.dart'; -import 'package:moor_generator/src/model/specified_dao.dart'; -import 'package:moor_generator/src/model/specified_database.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/model/sql_query.dart'; -import 'package:moor_generator/src/parser/column_parser.dart'; -import 'package:moor_generator/src/parser/moor/moor_analyzer.dart'; -import 'package:moor_generator/src/parser/sql/sql_parser.dart'; -import 'package:moor_generator/src/parser/sql/type_mapping.dart'; -import 'package:moor_generator/src/parser/table_parser.dart'; -import 'package:moor_generator/src/parser/use_dao_parser.dart'; -import 'package:moor_generator/src/parser/use_moor_parser.dart'; -import 'package:source_gen/source_gen.dart'; - -import 'errors.dart'; -import 'generator_state.dart'; -import 'options.dart'; -import 'writer.dart'; - -class GeneratorSession { - final GeneratorState state; - final ErrorStore errors = ErrorStore(); - final BuildStep step; - - final Writer writer = Writer(); - - TableParser _tableParser; - ColumnParser _columnParser; - - MoorOptions get options => state.options; - - GeneratorSession(this.state, this.step) { - _tableParser = TableParser(this); - _columnParser = ColumnParser(this); - } - - Future loadElementDeclaration( - Element element) async { - final resolvedLibrary = await element.library.session - .getResolvedLibraryByElement(element.library); - - return resolvedLibrary.getElementDeclaration(element); - } - - /// Parses a [SpecifiedDatabase] from the [ClassElement] which was annotated - /// with `@UseMoor` and the [annotation] reader that reads the `@UseMoor` - /// annotation. - Future parseDatabase( - ClassElement element, ConstantReader annotation) { - return UseMoorParser(this).parseDatabase(element, annotation); - } - - /// Parses a [SpecifiedDao] from a class declaration that has a `UseDao` - /// [annotation]. - Future parseDao( - ClassElement element, ConstantReader annotation) { - return UseDaoParser(this).parseDao(element, annotation); - } - - /// Resolves a [SpecifiedTable] for the class of each [DartType] in [types]. - /// The [initializedBy] element should be the piece of code that caused the - /// parsing (e.g. the database class that is annotated with `@UseMoor`). This - /// will allow for more descriptive error messages. - Future> parseTables( - Iterable types, Element initializedBy) { - return Future.wait(types.map((type) { - if (!state.tableTypeChecker.isAssignableFrom(type.element)) { - errors.add(MoorError( - critical: true, - message: 'The type $type is not a moor table', - affectedElement: initializedBy, - )); - return null; - } else { - return _tableParser.parse(type.element as ClassElement); - } - })).then((list) => List.from(list)); // make growable - } - - Future> resolveIncludes(Iterable paths) async { - final mapper = TypeMapper(); - final foundTables = []; - - for (var path in paths) { - final asset = AssetId.resolve(path, from: step.inputId); - String content; - try { - content = await step.readAsString(asset); - } catch (e) { - errors.add(MoorError( - critical: true, - message: 'The included file $path could not be found')); - } - - final parsed = await MoorAnalyzer(content).analyze(); - foundTables.addAll( - parsed.parsedFile.declaredTables.map((t) => t.extractTable(mapper))); - - for (var parseError in parsed.errors) { - errors.add(MoorError(message: "Can't parse sql in $path: $parseError")); - } - } - - return foundTables; - } - - /// Parses a column from a getter [e] declared inside a table class and its - /// resolved AST node [m]. - Future parseColumn(MethodDeclaration m, Element e) { - return Future.value(_columnParser.parse(m, e)); - } - - Future> parseQueries( - Map fromAnnotation, - List availableTables) { - // no queries declared, so there is no point in starting a sql engine - if (fromAnnotation.isEmpty) return Future.value([]); - - final parser = SqlParser(this, availableTables, fromAnnotation)..parse(); - - return Future.value(parser.foundQueries); - } -} diff --git a/moor_generator/lib/src/writer/database_writer.dart b/moor_generator/lib/src/writer/database_writer.dart index 174bcbc8..537ea97e 100644 --- a/moor_generator/lib/src/writer/database_writer.dart +++ b/moor_generator/lib/src/writer/database_writer.dart @@ -1,11 +1,8 @@ import 'package:moor_generator/src/model/sql_query.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:moor_generator/src/writer/query_writer.dart'; -import 'package:moor_generator/src/writer/result_set_writer.dart'; +import 'package:moor_generator/src/writer/utils/memoized_getter.dart'; import 'package:recase/recase.dart'; import 'package:moor_generator/src/model/specified_database.dart'; import 'package:moor_generator/src/writer/table_writer.dart'; -import 'utils.dart'; class DatabaseWriter { final SpecifiedDatabase db; @@ -19,13 +16,6 @@ class DatabaseWriter { TableWriter(table, session).writeInto(buffer); } - // Write additional classes to hold the result of custom queries - for (final query in db.queries) { - if (query is SqlSelectQuery && query.resultSet.matchingTable == null) { - ResultSetWriter(query).write(buffer); - } - } - // Write the database class final className = '_\$${db.fromClass.name}'; buffer.write('abstract class $className extends GeneratedDatabase {\n' diff --git a/moor_generator/lib/src/writer/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart similarity index 68% rename from moor_generator/lib/src/writer/query_writer.dart rename to moor_generator/lib/src/writer/queries/query_writer.dart index bf40cd6d..098d6fb2 100644 --- a/moor_generator/lib/src/writer/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -1,9 +1,11 @@ import 'dart:math' show max; +import 'package:moor_generator/src/backends/build/moor_builder.dart'; import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/sql_query.dart'; -import 'package:moor_generator/src/state/session.dart'; import 'package:moor_generator/src/utils/string_escaper.dart'; +import 'package:moor_generator/src/writer/queries/result_set_writer.dart'; +import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; import 'package:sqlparser/sqlparser.dart'; @@ -16,13 +18,18 @@ const highestAssignedIndexVar = '\$highestIndex'; /// should be included in a generated database or dao class. class QueryWriter { final SqlQuery query; - final GeneratorSession session; + final Scope scope; SqlSelectQuery get _select => query as SqlSelectQuery; UpdatingQuery get _update => query as UpdatingQuery; + MoorOptions get options => scope.writer.options; + StringBuffer _buffer; + final Set _writtenMappingMethods; - QueryWriter(this.query, this.session, this._writtenMappingMethods); + QueryWriter(this.query, this.scope, this._writtenMappingMethods) { + _buffer = scope.leaf(); + } /// The expanded sql that we insert into queries whenever an array variable /// appears. For the query "SELECT * FROM t WHERE x IN ?", we generate @@ -36,19 +43,25 @@ class QueryWriter { return 'expanded${v.dartParameterName}'; } - void writeInto(StringBuffer buffer) { + void writeInto() { if (query is SqlSelectQuery) { - _writeSelect(buffer); + final select = query as SqlSelectQuery; + if (select.resultSet.matchingTable == null) { + // query needs its own result set - write that now + final buffer = scope.findScopeOfLevel(DartScope.library).leaf(); + ResultSetWriter(select).write(buffer); + } + _writeSelect(); } else if (query is UpdatingQuery) { - _writeUpdatingQuery(buffer); + _writeUpdatingQuery(); } } - void _writeSelect(StringBuffer buffer) { - _writeMapping(buffer); - _writeSelectStatementCreator(buffer); - _writeOneTimeReader(buffer); - _writeStreamReader(buffer); + void _writeSelect() { + _writeMapping(); + _writeSelectStatementCreator(); + _writeOneTimeReader(); + _writeStreamReader(); } String _nameOfMappingMethod() { @@ -61,11 +74,11 @@ class QueryWriter { /// Writes a mapping method that turns a "QueryRow" into the desired custom /// return type. - void _writeMapping(StringBuffer buffer) { + void _writeMapping() { // avoid writing mapping methods twice if the same result class is written // more than once. if (!_writtenMappingMethods.contains(_nameOfMappingMethod())) { - buffer + _buffer ..write('${_select.resultClassName} ${_nameOfMappingMethod()}') ..write('(QueryRow row) {\n') ..write('return ${_select.resultClassName}('); @@ -84,35 +97,35 @@ class QueryWriter { code = '$field.mapToDart($code)'; } - buffer.write('$fieldName: $code,'); + _buffer.write('$fieldName: $code,'); } - buffer.write(');\n}\n'); + _buffer.write(');\n}\n'); _writtenMappingMethods.add(_nameOfMappingMethod()); } } /// Writes a method returning a `Selectable`, where `T` is the return type /// of the custom query. - void _writeSelectStatementCreator(StringBuffer buffer) { + void _writeSelectStatementCreator() { final returnType = 'Selectable<${_select.resultClassName}>'; final methodName = _nameOfCreationMethod(); - buffer.write('$returnType $methodName('); - _writeParameters(buffer); - buffer.write(') {\n'); + _buffer.write('$returnType $methodName('); + _writeParameters(); + _buffer.write(') {\n'); - _writeExpandedDeclarations(buffer); - buffer + _writeExpandedDeclarations(); + _buffer ..write('return (operateOn ?? this).') ..write('customSelectQuery(${_queryCode()}, '); - _writeVariables(buffer); - buffer.write(', '); - _writeReadsFrom(buffer); + _writeVariables(); + _buffer.write(', '); + _writeReadsFrom(); - buffer.write(').map('); - buffer.write(_nameOfMappingMethod()); - buffer.write(');\n}\n'); + _buffer.write(').map('); + _buffer.write(_nameOfMappingMethod()); + _buffer.write(');\n}\n'); } /* @@ -122,34 +135,35 @@ class QueryWriter { } */ - void _writeOneTimeReader(StringBuffer buffer) { - buffer.write('Future> ${query.name}('); - _writeParameters(buffer); - buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); - _writeUseParameters(buffer); - buffer.write(').get();\n}\n'); + void _writeOneTimeReader() { + _buffer.write('Future> ${query.name}('); + _writeParameters(); + _buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); + _writeUseParameters(); + _buffer.write(').get();\n}\n'); } - void _writeStreamReader(StringBuffer buffer) { + void _writeStreamReader() { final upperQueryName = ReCase(query.name).pascalCase; String methodName; // turning the query name into pascal case will remove underscores, add the // "private" modifier back in if needed - if (session.options.fixPrivateWatchMethods && query.name.startsWith('_')) { + if (scope.writer.options.fixPrivateWatchMethods && + query.name.startsWith('_')) { methodName = '_watch$upperQueryName'; } else { methodName = 'watch$upperQueryName'; } - buffer.write('Stream> $methodName('); - _writeParameters(buffer, dontOverrideEngine: true); - buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); - _writeUseParameters(buffer, dontUseEngine: true); - buffer.write(').watch();\n}\n'); + _buffer.write('Stream> $methodName('); + _writeParameters(dontOverrideEngine: true); + _buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); + _writeUseParameters(dontUseEngine: true); + _buffer.write(').watch();\n}\n'); } - void _writeUpdatingQuery(StringBuffer buffer) { + void _writeUpdatingQuery() { /* Future test() { return customUpdate('', variables: [], updates: {}); @@ -157,24 +171,23 @@ class QueryWriter { */ final implName = _update.isInsert ? 'customInsert' : 'customUpdate'; - buffer.write('Future ${query.name}('); - _writeParameters(buffer); - buffer.write(') {\n'); + _buffer.write('Future ${query.name}('); + _writeParameters(); + _buffer.write(') {\n'); - _writeExpandedDeclarations(buffer); - buffer + _writeExpandedDeclarations(); + _buffer ..write('return (operateOn ?? this).') ..write('$implName(${_queryCode()},'); - _writeVariables(buffer); - buffer.write(','); - _writeUpdates(buffer); + _writeVariables(); + _buffer.write(','); + _writeUpdates(); - buffer..write(',);\n}\n'); + _buffer..write(',);\n}\n'); } - void _writeParameters(StringBuffer buffer, - {bool dontOverrideEngine = false}) { + void _writeParameters({bool dontOverrideEngine = false}) { final paramList = query.variables.map((v) { var dartType = dartTypeNames[v.type]; if (v.isArray) { @@ -183,13 +196,13 @@ class QueryWriter { return '$dartType ${v.dartParameterName}'; }).join(', '); - buffer.write(paramList); + _buffer.write(paramList); // write named optional parameter to configure the query engine used to // execute the statement, if (!dontOverrideEngine) { - if (query.variables.isNotEmpty) buffer.write(', '); - buffer.write('{@Deprecated(${asDartLiteral(queryEngineWarningDesc)}) ' + if (query.variables.isNotEmpty) _buffer.write(', '); + _buffer.write('{@Deprecated(${asDartLiteral(queryEngineWarningDesc)}) ' 'QueryEngine operateOn}'); } } @@ -197,11 +210,11 @@ class QueryWriter { /// Writes code that uses the parameters as declared by [_writeParameters], /// assuming that for each parameter, a variable with the same name exists /// in the current scope. - void _writeUseParameters(StringBuffer into, {bool dontUseEngine = false}) { - into.write(query.variables.map((v) => v.dartParameterName).join(', ')); + void _writeUseParameters({bool dontUseEngine = false}) { + _buffer.write(query.variables.map((v) => v.dartParameterName).join(', ')); if (!dontUseEngine) { - if (query.variables.isNotEmpty) into.write(', '); - into.write('operateOn: operateOn'); + if (query.variables.isNotEmpty) _buffer.write(', '); + _buffer.write('operateOn: operateOn'); } } @@ -217,7 +230,7 @@ class QueryWriter { // "vars" variable twice. To do this, a local var called "$currentVarIndex" // keeps track of the highest variable number assigned. - void _writeExpandedDeclarations(StringBuffer buffer) { + void _writeExpandedDeclarations() { var indexCounterWasDeclared = false; var highestIndexBeforeArray = 0; @@ -228,12 +241,12 @@ class QueryWriter { // add +1 because that's going to be the first index of the expanded // array final firstVal = highestIndexBeforeArray + 1; - buffer.write('var $highestAssignedIndexVar = $firstVal;'); + _buffer.write('var $highestAssignedIndexVar = $firstVal;'); indexCounterWasDeclared = true; } // final expandedvar1 = $expandVar(, ); - buffer + _buffer ..write('final ') ..write(_expandedName(variable)) ..write(' = ') @@ -244,7 +257,7 @@ class QueryWriter { ..write('.length);\n'); // increase highest index for the next array - buffer + _buffer ..write('$highestAssignedIndexVar += ') ..write(variable.dartParameterName) ..write('.length;'); @@ -256,8 +269,8 @@ class QueryWriter { } } - void _writeVariables(StringBuffer buffer) { - buffer..write('variables: ['); + void _writeVariables() { + _buffer..write('variables: ['); for (var variable in query.variables) { // for a regular variable: Variable.withInt(x), @@ -266,15 +279,15 @@ class QueryWriter { final name = variable.dartParameterName; if (variable.isArray) { - buffer.write('for (var \$ in $name) $constructor(\$)'); + _buffer.write('for (var \$ in $name) $constructor(\$)'); } else { - buffer.write('$constructor($name)'); + _buffer.write('$constructor($name)'); } - buffer.write(','); + _buffer.write(','); } - buffer..write(']'); + _buffer..write(']'); } /// Returns a Dart string literal representing the query after variables have @@ -295,29 +308,29 @@ class QueryWriter { .singleWhere((f) => f.variable.resolvedIndex == sqlVar.resolvedIndex); if (!moorVar.isArray) continue; - // write everything that comes before this var into the buffer + // write everything that comes before this var into the_buffer final currentIndex = sqlVar.firstPosition; final queryPart = query.sql.substring(lastIndex, currentIndex); - buffer.write(escapeForDart(queryPart)); + _buffer.write(escapeForDart(queryPart)); lastIndex = sqlVar.lastPosition; // write the ($expandedVar) par - buffer.write('(\$${_expandedName(moorVar)})'); + _buffer.write('(\$${_expandedName(moorVar)})'); } // write the final part after the last variable, plus the ending ' - buffer..write(escapeForDart(query.sql.substring(lastIndex)))..write("'"); + _buffer..write(escapeForDart(query.sql.substring(lastIndex)))..write("'"); return buffer.toString(); } - void _writeReadsFrom(StringBuffer buffer) { + void _writeReadsFrom() { final from = _select.readsFrom.map((t) => t.tableFieldName).join(', '); - buffer..write('readsFrom: {')..write(from)..write('}'); + _buffer..write('readsFrom: {')..write(from)..write('}'); } - void _writeUpdates(StringBuffer buffer) { + void _writeUpdates() { final from = _update.updates.map((t) => t.tableFieldName).join(', '); - buffer..write('updates: {')..write(from)..write('}'); + _buffer..write('updates: {')..write(from)..write('}'); } } diff --git a/moor_generator/lib/src/writer/result_set_writer.dart b/moor_generator/lib/src/writer/queries/result_set_writer.dart similarity index 93% rename from moor_generator/lib/src/writer/result_set_writer.dart rename to moor_generator/lib/src/writer/queries/result_set_writer.dart index 39fcdd4e..88ea7d63 100644 --- a/moor_generator/lib/src/writer/result_set_writer.dart +++ b/moor_generator/lib/src/writer/queries/result_set_writer.dart @@ -1,6 +1,7 @@ import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/sql_query.dart'; +/// Writes a class holding the result of an sql query into Dart. class ResultSetWriter { final SqlSelectQuery query; diff --git a/moor_generator/lib/src/writer/data_class_writer.dart b/moor_generator/lib/src/writer/tables/data_class_writer.dart similarity index 97% rename from moor_generator/lib/src/writer/data_class_writer.dart rename to moor_generator/lib/src/writer/tables/data_class_writer.dart index 1f70aa7b..a6eabb55 100644 --- a/moor_generator/lib/src/writer/data_class_writer.dart +++ b/moor_generator/lib/src/writer/tables/data_class_writer.dart @@ -1,13 +1,17 @@ import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/state/session.dart'; import 'package:moor_generator/src/writer/utils/hash_code.dart'; +import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; class DataClassWriter { final SpecifiedTable table; - final GeneratorSession session; + final Scope scope; - DataClassWriter(this.table, this.session); + StringBuffer _buffer; + + DataClassWriter(this.table, this.scope) { + _buffer = scope.leaf(); + } void writeInto(StringBuffer buffer) { buffer.write( diff --git a/moor_generator/lib/src/writer/table_writer.dart b/moor_generator/lib/src/writer/tables/table_writer.dart similarity index 99% rename from moor_generator/lib/src/writer/table_writer.dart rename to moor_generator/lib/src/writer/tables/table_writer.dart index 05996443..74910861 100644 --- a/moor_generator/lib/src/writer/table_writer.dart +++ b/moor_generator/lib/src/writer/tables/table_writer.dart @@ -4,7 +4,7 @@ import 'package:moor_generator/src/state/session.dart'; import 'package:moor_generator/src/utils/string_escaper.dart'; import 'package:moor_generator/src/writer/data_class_writer.dart'; import 'package:moor_generator/src/writer/update_companion_writer.dart'; -import 'package:moor_generator/src/writer/utils.dart'; +import 'package:moor_generator/src/writer/memoized_getter.dart'; class TableWriter { final SpecifiedTable table; diff --git a/moor_generator/lib/src/writer/update_companion_writer.dart b/moor_generator/lib/src/writer/tables/update_companion_writer.dart similarity index 100% rename from moor_generator/lib/src/writer/update_companion_writer.dart rename to moor_generator/lib/src/writer/tables/update_companion_writer.dart diff --git a/moor_generator/lib/src/writer/utils.dart b/moor_generator/lib/src/writer/utils/memoized_getter.dart similarity index 100% rename from moor_generator/lib/src/writer/utils.dart rename to moor_generator/lib/src/writer/utils/memoized_getter.dart diff --git a/moor_generator/lib/src/state/writer.dart b/moor_generator/lib/src/writer/writer.dart similarity index 62% rename from moor_generator/lib/src/state/writer.dart rename to moor_generator/lib/src/writer/writer.dart index b55a87dd..1fad3f04 100644 --- a/moor_generator/lib/src/state/writer.dart +++ b/moor_generator/lib/src/writer/writer.dart @@ -1,4 +1,5 @@ import 'package:meta/meta.dart'; +import 'package:moor_generator/src/backends/build/moor_builder.dart'; /// Manages a tree structure which we use to generate code. /// @@ -10,6 +11,9 @@ import 'package:meta/meta.dart'; /// passing a [Scope] we will always be able to write code in a parent scope. class Writer { final Scope _root = Scope(parent: null); + final MoorOptions options; + + Writer(this.options); String writeGenerated() => _leafNodes(_root).join(); @@ -41,8 +45,13 @@ abstract class _Node { /// we just pass a single [StringBuffer] around, this is annoying to manage. class Scope extends _Node { final List<_Node> _children = []; + final DartScope scope; + final Writer writer; - Scope({@required Scope parent}) : super(parent); + Scope({@required Scope parent, Writer writer}) + : scope = parent?.scope?.nextLevel ?? DartScope.library, + writer = writer ?? parent?.writer, + super(parent); Scope get root { var found = this; @@ -52,6 +61,19 @@ class Scope extends _Node { return found; } + Iterable get _thisAndParents sync* { + var scope = this; + do { + yield scope; + scope = scope.parent; + } while (scope != null); + } + + Scope findScopeOfLevel(DartScope level) { + return _thisAndParents + .firstWhere((scope) => scope.scope.isSuperScope(level)); + } + Scope child() { final child = Scope(parent: this); _children.add(child); @@ -70,3 +92,27 @@ class _LeafNode extends _Node { _LeafNode(Scope parent) : super(parent); } + +class DartScope { + static const DartScope library = DartScope._(0); + static const DartScope topLevelMember = DartScope._(1); + static const DartScope inner = DartScope._(2); + + static const List values = [library, topLevelMember, inner]; + + final int _id; + + const DartScope._(this._id); + + DartScope get nextLevel { + if (_id == values.length - 1) { + // already in innermost level + return this; + } + return values[_id + 1]; + } + + bool isSuperScope(DartScope other) { + return other._id >= _id; + } +} From 74dc2e5404dad0abac6ac4b2ccba10bc691525cd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 2 Sep 2019 20:28:57 +0200 Subject: [PATCH 044/117] Finish refactoring of moor_generator --- moor_generator/lib/src/analyzer/session.dart | 25 ++- moor_generator/lib/src/backends/backend.dart | 8 +- .../lib/src/backends/build/build_backend.dart | 27 ++-- .../build/generators}/dao_generator.dart | 42 ++--- .../build/generators/moor_generator.dart | 44 ++++++ .../lib/src/backends/build/moor_builder.dart | 32 +++- moor_generator/lib/src/moor_generator.dart | 54 ------- .../lib/src/writer/database_writer.dart | 26 ++-- .../lib/src/writer/queries/query_writer.dart | 2 +- .../src/writer/tables/data_class_writer.dart | 103 ++++++------- .../lib/src/writer/tables/table_writer.dart | 121 ++++++++------- .../tables/update_companion_writer.dart | 72 ++++----- moor_generator/pubspec.yaml | 1 + .../test/analyzer/dart/dart_test.dart | 6 +- .../test/analyzer/dart/table_parser_test.dart | 5 +- .../test/parser/moor/moor_parser_test.dart | 14 +- moor_generator/test/parser/parser_test.dart | 144 ------------------ .../test/parser/sql/query_handler_test.dart | 4 +- moor_generator/test/utils/test_backend.dart | 22 +-- .../writer/utils/hash_code_test.dart | 0 20 files changed, 327 insertions(+), 425 deletions(-) rename moor_generator/lib/src/{ => backends/build/generators}/dao_generator.dart (56%) create mode 100644 moor_generator/lib/src/backends/build/generators/moor_generator.dart delete mode 100644 moor_generator/lib/src/moor_generator.dart delete mode 100644 moor_generator/test/parser/parser_test.dart rename moor_generator/test/{parser => }/writer/utils/hash_code_test.dart (100%) diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index bbb51cb7..7831bfdf 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -21,13 +21,13 @@ import 'package:source_gen/source_gen.dart'; class MoorSession { MoorSession(); - Future startDartTask(BackendTask backendTask, {String uri}) async { + Future startDartTask(BackendTask backendTask, {Uri uri}) async { final input = uri ?? backendTask.entrypoint; final library = await backendTask.resolveDart(input); return DartTask(this, backendTask, library); } - Future startMoorTask(BackendTask backendTask, {String uri}) async { + Future startMoorTask(BackendTask backendTask, {Uri uri}) async { final input = uri ?? backendTask.entrypoint; final source = await backendTask.readMoor(input); return MoorTask(backendTask, this, source); @@ -46,6 +46,23 @@ abstract class FileTask { void reportError(MoorError error) => errors.report(error); FutureOr compute(); + + void printErrors() { + /* + * if (session.errors.errors.isNotEmpty) { + print('Warning: There were some errors while running ' + 'moor_generator on ${buildStep.inputId.path}:'); + + for (var error in session.errors.errors) { + print(error.message); + + if (error.affectedElement != null) { + final span = spanForElement(error.affectedElement); + print('${span.start.toolString}\n${span.highlight()}'); + } + } + } */ + } } /// Session used to parse a Dart file and extract table information. @@ -104,8 +121,8 @@ class DartTask extends FileTask { /// Reads all tables declared in sql by a `.moor` file in [paths]. Future> resolveIncludes(Iterable paths) { - return Stream.fromFutures( - paths.map((path) => session.startMoorTask(backendTask, uri: path))) + return Stream.fromFutures(paths.map( + (path) => session.startMoorTask(backendTask, uri: Uri.parse(path)))) .asyncMap((task) => task.compute()) .expand((file) => file.declaredTables) .toList(); diff --git a/moor_generator/lib/src/backends/backend.dart b/moor_generator/lib/src/backends/backend.dart index 5ab51df8..4a5ec059 100644 --- a/moor_generator/lib/src/backends/backend.dart +++ b/moor_generator/lib/src/backends/backend.dart @@ -1,4 +1,5 @@ import 'package:analyzer/dart/element/element.dart'; +import 'package:logging/logging.dart'; import 'package:moor_generator/src/analyzer/session.dart'; /// A backend for the moor generator. @@ -12,8 +13,9 @@ abstract class Backend { /// Used to analyze a single file via ([entrypoint]). The other methods can be /// used to read imports used by the other files. abstract class BackendTask { - String get entrypoint; + Uri get entrypoint; + Logger get log; - Future resolveDart(String path); - Future readMoor(String path); + Future resolveDart(Uri uri); + Future readMoor(Uri uri); } diff --git a/moor_generator/lib/src/backends/build/build_backend.dart b/moor_generator/lib/src/backends/build/build_backend.dart index f511cd7c..825e1c3e 100644 --- a/moor_generator/lib/src/backends/build/build_backend.dart +++ b/moor_generator/lib/src/backends/build/build_backend.dart @@ -1,8 +1,14 @@ import 'package:analyzer/dart/element/element.dart'; -import 'package:build/build.dart'; +import 'package:build/build.dart' hide log; +import 'package:build/build.dart' as build show log; +import 'package:logging/logging.dart'; import 'package:moor_generator/src/backends/backend.dart'; -class BuildBackend extends Backend {} +class BuildBackend extends Backend { + BuildBackendTask createTask(BuildStep step) { + return BuildBackendTask(step); + } +} class BuildBackendTask extends BackendTask { final BuildStep step; @@ -10,19 +16,22 @@ class BuildBackendTask extends BackendTask { BuildBackendTask(this.step); @override - String get entrypoint => step.inputId.path; + Uri get entrypoint => step.inputId.uri; - AssetId _resolve(String uri) { - return AssetId.resolve(uri, from: step.inputId); + AssetId _resolve(Uri uri) { + return AssetId.resolve(uri.toString(), from: step.inputId); } @override - Future readMoor(String path) { - return step.readAsString(_resolve(path)); + Future readMoor(Uri uri) { + return step.readAsString(_resolve(uri)); } @override - Future resolveDart(String path) { - return step.resolver.libraryFor(_resolve(path)); + Future resolveDart(Uri uri) { + return step.resolver.libraryFor(_resolve(uri)); } + + @override + Logger get log => build.log; } diff --git a/moor_generator/lib/src/dao_generator.dart b/moor_generator/lib/src/backends/build/generators/dao_generator.dart similarity index 56% rename from moor_generator/lib/src/dao_generator.dart rename to moor_generator/lib/src/backends/build/generators/dao_generator.dart index 0ecf9761..2abaf0e7 100644 --- a/moor_generator/lib/src/dao_generator.dart +++ b/moor_generator/lib/src/backends/build/generators/dao_generator.dart @@ -1,24 +1,20 @@ import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart'; import 'package:moor/moor.dart'; -import 'package:moor_generator/src/state/generator_state.dart'; -import 'package:moor_generator/src/state/options.dart'; -import 'package:moor_generator/src/writer/query_writer.dart'; -import 'package:moor_generator/src/writer/result_set_writer.dart'; +import 'package:moor_generator/src/backends/build/moor_builder.dart'; +import 'package:moor_generator/src/writer/queries/query_writer.dart'; +import 'package:moor_generator/src/writer/writer.dart'; import 'package:source_gen/source_gen.dart'; -import 'model/sql_query.dart'; - -class DaoGenerator extends GeneratorForAnnotation { - final MoorOptions options; - - DaoGenerator(this.options); +class DaoGenerator extends GeneratorForAnnotation + implements BaseGenerator { + @override + MoorBuilder builder; @override generateForAnnotatedElement( Element element, ConstantReader annotation, BuildStep buildStep) async { - final state = useState(() => GeneratorState(options)); - final session = state.startSession(buildStep); + final task = await builder.createDartTask(buildStep); if (element is! ClassElement) { throw InvalidGenerationSourceError( @@ -27,7 +23,7 @@ class DaoGenerator extends GeneratorForAnnotation { } final targetClass = element as ClassElement; - final parsedDao = await session.parseDao(targetClass, annotation); + final parsedDao = await task.parseDao(targetClass, annotation); final dbType = targetClass.supertype; if (dbType.name != 'DatabaseAccessor') { @@ -46,33 +42,27 @@ class DaoGenerator extends GeneratorForAnnotation { } // finally, we can write the mixin - final buffer = StringBuffer(); + final writer = Writer(builder.options); + final classScope = writer.child(); final daoName = targetClass.displayName; - buffer.write('mixin _\$${daoName}Mixin on ' + classScope.leaf().write('mixin _\$${daoName}Mixin on ' 'DatabaseAccessor<${dbImpl.displayName}> {\n'); for (var table in parsedDao.tables) { final infoType = table.tableInfoName; final getterName = table.tableFieldName; - buffer.write('$infoType get $getterName => db.$getterName;\n'); + classScope.leaf().write('$infoType get $getterName => db.$getterName;\n'); } final writtenMappingMethods = {}; for (var query in parsedDao.queries) { - QueryWriter(query, session, writtenMappingMethods).writeInto(buffer); + QueryWriter(query, classScope.child(), writtenMappingMethods).write(); } - buffer.write('}'); + classScope.leaf().write('}'); - // if the queries introduced additional classes, also write those - for (final query in parsedDao.queries) { - if (query is SqlSelectQuery && query.resultSet.matchingTable == null) { - ResultSetWriter(query).write(buffer); - } - } - - return buffer.toString(); + return writer.writeGenerated(); } } diff --git a/moor_generator/lib/src/backends/build/generators/moor_generator.dart b/moor_generator/lib/src/backends/build/generators/moor_generator.dart new file mode 100644 index 00000000..6adc3867 --- /dev/null +++ b/moor_generator/lib/src/backends/build/generators/moor_generator.dart @@ -0,0 +1,44 @@ +import 'package:moor/moor.dart'; +import 'package:analyzer/dart/element/element.dart'; +import 'package:build/build.dart'; +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/backends/build/moor_builder.dart'; +import 'package:moor_generator/src/writer/database_writer.dart'; +import 'package:moor_generator/src/writer/writer.dart'; +import 'package:source_gen/source_gen.dart'; + +class MoorGenerator extends GeneratorForAnnotation + implements BaseGenerator { + @override + MoorBuilder builder; + + @override + generateForAnnotatedElement( + Element element, ConstantReader annotation, BuildStep buildStep) async { + final task = await builder.createDartTask(buildStep); + + if (element is! ClassElement) { + task.reportError(ErrorInDartCode( + severity: Severity.criticalError, + message: 'This annotation can only be used on classes', + affectedElement: element, + )); + } + + final database = + await task.parseDatabase(element as ClassElement, annotation); + + task.printErrors(); + + if (database.tables.isEmpty) return ''; + + final writer = Writer(builder.options); + writer + .leaf() + .write('// ignore_for_file: unnecessary_brace_in_string_interps\n'); + + DatabaseWriter(database, writer.child()).write(); + + return writer.writeGenerated(); + } +} diff --git a/moor_generator/lib/src/backends/build/moor_builder.dart b/moor_generator/lib/src/backends/build/moor_builder.dart index 66698c07..4ff994e5 100644 --- a/moor_generator/lib/src/backends/build/moor_builder.dart +++ b/moor_generator/lib/src/backends/build/moor_builder.dart @@ -1,25 +1,43 @@ import 'package:build/build.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/backends/build/build_backend.dart'; -import 'package:moor_generator/src/dao_generator.dart'; -import 'package:moor_generator/src/moor_generator.dart'; +import 'package:moor_generator/src/backends/build/generators/dao_generator.dart'; +import 'package:moor_generator/src/backends/build/generators/moor_generator.dart'; import 'package:source_gen/source_gen.dart'; part 'options.dart'; class MoorBuilder extends SharedPartBuilder { final BuildBackend backend = BuildBackend(); + final MoorOptions options; factory MoorBuilder(BuilderOptions options) { final parsedOptions = MoorOptions.fromBuilder(options.config); - final generators = [ - MoorGenerator(parsedOptions), - DaoGenerator(parsedOptions), + final generators = [ + MoorGenerator(), + DaoGenerator(), ]; - return MoorBuilder._(generators, 'moor'); + final builder = MoorBuilder._(generators, 'moor', parsedOptions); + + for (var generator in generators.cast()) { + generator.builder = builder; + } + + return builder; } - MoorBuilder._(List generators, String name) + MoorBuilder._(List generators, String name, this.options) : super(generators, name); + + Future createDartTask(BuildStep step) async { + final backendTask = backend.createTask(step); + return await backend.session + .startDartTask(backendTask, uri: step.inputId.uri); + } +} + +abstract class BaseGenerator { + MoorBuilder builder; } diff --git a/moor_generator/lib/src/moor_generator.dart b/moor_generator/lib/src/moor_generator.dart deleted file mode 100644 index cca7f3bd..00000000 --- a/moor_generator/lib/src/moor_generator.dart +++ /dev/null @@ -1,54 +0,0 @@ -import 'package:moor/moor.dart'; -import 'package:analyzer/dart/element/element.dart'; -import 'package:build/build.dart'; -import 'package:moor_generator/src/state/errors.dart'; -import 'package:moor_generator/src/state/generator_state.dart'; -import 'package:moor_generator/src/state/options.dart'; -import 'package:moor_generator/src/writer/database_writer.dart'; -import 'package:source_gen/source_gen.dart'; - -class MoorGenerator extends GeneratorForAnnotation { - final MoorOptions options; - MoorGenerator(this.options); - - @override - generateForAnnotatedElement( - Element element, ConstantReader annotation, BuildStep buildStep) async { - final state = useState(() => GeneratorState(options)); - final session = state.startSession(buildStep); - - if (element is! ClassElement) { - session.errors.add(MoorError( - critical: true, - message: 'This annotation can only be used on classes', - affectedElement: element, - )); - } - - final database = - await session.parseDatabase(element as ClassElement, annotation); - - if (session.errors.errors.isNotEmpty) { - print('Warning: There were some errors while running ' - 'moor_generator on ${buildStep.inputId.path}:'); - - for (var error in session.errors.errors) { - print(error.message); - - if (error.affectedElement != null) { - final span = spanForElement(error.affectedElement); - print('${span.start.toolString}\n${span.highlight()}'); - } - } - } - - if (database.tables.isEmpty) return ''; - - final buffer = StringBuffer() - ..write('// ignore_for_file: unnecessary_brace_in_string_interps\n'); - - DatabaseWriter(database, session).write(buffer); - - return buffer.toString(); - } -} diff --git a/moor_generator/lib/src/writer/database_writer.dart b/moor_generator/lib/src/writer/database_writer.dart index 537ea97e..ef9a9784 100644 --- a/moor_generator/lib/src/writer/database_writer.dart +++ b/moor_generator/lib/src/writer/database_writer.dart @@ -1,24 +1,28 @@ -import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:moor_generator/src/writer/queries/query_writer.dart'; +import 'package:moor_generator/src/writer/tables/table_writer.dart'; import 'package:moor_generator/src/writer/utils/memoized_getter.dart'; +import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; import 'package:moor_generator/src/model/specified_database.dart'; -import 'package:moor_generator/src/writer/table_writer.dart'; class DatabaseWriter { final SpecifiedDatabase db; - final GeneratorSession session; + final Scope scope; - DatabaseWriter(this.db, this.session); + DatabaseWriter(this.db, this.scope); - void write(StringBuffer buffer) { + void write() { // Write referenced tables for (final table in db.tables) { - TableWriter(table, session).writeInto(buffer); + TableWriter(table, scope.child()).writeInto(); } // Write the database class + final dbScope = scope.child(); + final className = '_\$${db.fromClass.name}'; - buffer.write('abstract class $className extends GeneratedDatabase {\n' + dbScope.leaf().write( + 'abstract class $className extends GeneratedDatabase {\n' '$className(QueryExecutor e) : super(const SqlTypeSystem.withDefaults(), e); \n'); final tableGetters = []; @@ -28,7 +32,7 @@ class DatabaseWriter { final tableClassName = table.tableInfoName; writeMemoizedGetter( - buffer: buffer, + buffer: dbScope.leaf(), getterName: table.tableFieldName, returnType: tableClassName, code: '$tableClassName(this)', @@ -42,7 +46,7 @@ class DatabaseWriter { final databaseImplName = db.fromClass.name; writeMemoizedGetter( - buffer: buffer, + buffer: dbScope.leaf(), getterName: getterName, returnType: typeName, code: '$typeName(this as $databaseImplName)', @@ -52,11 +56,11 @@ class DatabaseWriter { // Write implementation for query methods final writtenMappingMethods = {}; for (var query in db.queries) { - QueryWriter(query, session, writtenMappingMethods).writeInto(buffer); + QueryWriter(query, dbScope.child(), writtenMappingMethods).write(); } // Write List of tables, close bracket for class - buffer + dbScope.leaf() ..write('@override\nList get allTables => [') ..write(tableGetters.join(',')) ..write('];\n}'); diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 098d6fb2..078ffbd4 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -43,7 +43,7 @@ class QueryWriter { return 'expanded${v.dartParameterName}'; } - void writeInto() { + void write() { if (query is SqlSelectQuery) { final select = query as SqlSelectQuery; if (select.resultSet.matchingTable == null) { diff --git a/moor_generator/lib/src/writer/tables/data_class_writer.dart b/moor_generator/lib/src/writer/tables/data_class_writer.dart index a6eabb55..e0232bec 100644 --- a/moor_generator/lib/src/writer/tables/data_class_writer.dart +++ b/moor_generator/lib/src/writer/tables/data_class_writer.dart @@ -13,17 +13,18 @@ class DataClassWriter { _buffer = scope.leaf(); } - void writeInto(StringBuffer buffer) { - buffer.write( + void write() { + _buffer.write( 'class ${table.dartTypeName} extends DataClass implements Insertable<${table.dartTypeName}> {\n'); // write individual fields for (var column in table.columns) { - buffer.write('final ${column.dartTypeName} ${column.dartGetterName}; \n'); + _buffer + .write('final ${column.dartTypeName} ${column.dartGetterName}; \n'); } // write constructor with named optional fields - buffer + _buffer ..write(table.dartTypeName) ..write('({') ..write(table.columns.map((column) { @@ -36,27 +37,27 @@ class DataClassWriter { ..write('});'); // Also write parsing factory - _writeMappingConstructor(buffer); + _writeMappingConstructor(); // And a serializer and deserializer method - _writeFromJson(buffer); - _writeToJson(buffer); - _writeCompanionOverride(buffer); + _writeFromJson(); + _writeToJson(); + _writeCompanionOverride(); // And a convenience method to copy data from this class. - _writeCopyWith(buffer); + _writeCopyWith(); - _writeToString(buffer); - _writeHashCode(buffer); + _writeToString(); + _writeHashCode(); // override == // return identical(this, other) || (other is DataClass && other.id == id && ...) - buffer + _buffer ..write('@override\nbool operator ==(other) => ') ..write('identical(this, other) || (other is ${table.dartTypeName}'); if (table.columns.isNotEmpty) { - buffer + _buffer ..write('&&') ..write(table.columns.map((c) { final getter = c.dartGetterName; @@ -66,13 +67,13 @@ class DataClassWriter { } // finish overrides method and class declaration - buffer.write(');\n}'); + _buffer.write(');\n}'); } - void _writeMappingConstructor(StringBuffer buffer) { + void _writeMappingConstructor() { final dataClassName = table.dartTypeName; - buffer + _buffer ..write('factory $dataClassName.fromData') ..write('(Map data, GeneratedDatabase db, ') ..write('{String prefix}) {\n') @@ -86,12 +87,12 @@ class DataClassWriter { final resolver = '${ReCase(usedType).camelCase}Type'; dartTypeToResolver[usedType] = resolver; - buffer + _buffer .write('final $resolver = db.typeSystem.forDartType<$usedType>();\n'); } // finally, the mighty constructor invocation: - buffer.write('return $dataClassName('); + _buffer.write('return $dataClassName('); for (var column in table.columns) { // id: intType.mapFromDatabaseResponse(data["id]) @@ -110,16 +111,16 @@ class DataClassWriter { loadType = '$loaded.mapToDart($loadType)'; } - buffer.write('$getter: $loadType,'); + _buffer.write('$getter: $loadType,'); } - buffer.write(');}\n'); + _buffer.write(');}\n'); } - void _writeFromJson(StringBuffer buffer) { + void _writeFromJson() { final dataClassName = table.dartTypeName; - buffer + _buffer ..write('factory $dataClassName.fromJson(' 'Map json,' '{ValueSerializer serializer = const ValueSerializer.defaults()}' @@ -131,14 +132,14 @@ class DataClassWriter { final jsonKey = column.jsonKey; final type = column.dartTypeName; - buffer.write("$getter: serializer.fromJson<$type>(json['$jsonKey']),"); + _buffer.write("$getter: serializer.fromJson<$type>(json['$jsonKey']),"); } - buffer.write(');}\n'); + _buffer.write(');}\n'); - if (session.options.generateFromJsonStringConstructor) { + if (scope.writer.options.generateFromJsonStringConstructor) { // also generate a constructor that only takes a json string - buffer.write('factory $dataClassName.fromJsonString(String encodedJson, ' + _buffer.write('factory $dataClassName.fromJsonString(String encodedJson, ' '{ValueSerializer serializer = const ValueSerializer.defaults()}) => ' '$dataClassName.fromJson(' 'DataClass.parseJson(encodedJson) as Map, ' @@ -146,8 +147,8 @@ class DataClassWriter { } } - void _writeToJson(StringBuffer buffer) { - buffer.write('@override Map toJson(' + void _writeToJson() { + _buffer.write('@override Map toJson(' '{ValueSerializer serializer = const ValueSerializer.defaults()}) {' '\n return {'); @@ -157,40 +158,40 @@ class DataClassWriter { final needsThis = getter == 'serializer'; final value = needsThis ? 'this.$getter' : getter; - buffer + _buffer .write("'$name': serializer.toJson<${column.dartTypeName}>($value),"); } - buffer.write('};}'); + _buffer.write('};}'); } - void _writeCopyWith(StringBuffer buffer) { + void _writeCopyWith() { final dataClassName = table.dartTypeName; - buffer.write('$dataClassName copyWith({'); + _buffer.write('$dataClassName copyWith({'); for (var i = 0; i < table.columns.length; i++) { final column = table.columns[i]; final last = i == table.columns.length - 1; - buffer.write('${column.dartTypeName} ${column.dartGetterName}'); + _buffer.write('${column.dartTypeName} ${column.dartGetterName}'); if (!last) { - buffer.write(','); + _buffer.write(','); } } - buffer.write('}) => $dataClassName('); + _buffer.write('}) => $dataClassName('); for (var column in table.columns) { // we also have a method parameter called like the getter, so we can use // field: field ?? this.field final getter = column.dartGetterName; - buffer.write('$getter: $getter ?? this.$getter,'); + _buffer.write('$getter: $getter ?? this.$getter,'); } - buffer.write(');'); + _buffer.write(');'); } - void _writeToString(StringBuffer buffer) { + void _writeToString() { /* @override String toString() { @@ -202,7 +203,7 @@ class DataClassWriter { } */ - buffer + _buffer ..write('@override\nString toString() {') ..write("return (StringBuffer('${table.dartTypeName}(')"); @@ -210,36 +211,36 @@ class DataClassWriter { final column = table.columns[i]; final getterName = column.dartGetterName; - buffer.write("..write('$getterName: \$$getterName"); - if (i != table.columns.length - 1) buffer.write(', '); + _buffer.write("..write('$getterName: \$$getterName"); + if (i != table.columns.length - 1) _buffer.write(', '); - buffer.write("')"); + _buffer.write("')"); } - buffer..write("..write(')')).toString();")..write('\}\n'); + _buffer..write("..write(')')).toString();")..write('\}\n'); } - void _writeHashCode(StringBuffer buffer) { - buffer.write('@override\n int get hashCode => '); + void _writeHashCode() { + _buffer.write('@override\n int get hashCode => '); final fields = table.columns.map((c) => c.dartGetterName).toList(); - HashCodeWriter().writeHashCode(fields, buffer); - buffer.write(';'); + HashCodeWriter().writeHashCode(fields, _buffer); + _buffer.write(';'); } - void _writeCompanionOverride(StringBuffer buffer) { + void _writeCompanionOverride() { // T createCompanion(bool nullToAbsent) final companionClass = table.updateCompanionName; - buffer.write('@override\nT createCompanion>(' 'bool nullToAbsent) {\n return $companionClass('); for (var column in table.columns) { final getter = column.dartGetterName; - buffer.write('$getter: $getter == null && nullToAbsent ? ' + _buffer.write('$getter: $getter == null && nullToAbsent ? ' 'const Value.absent() : Value($getter),'); } - buffer.write(') as T;}\n'); + _buffer.write(') as T;}\n'); } } diff --git a/moor_generator/lib/src/writer/tables/table_writer.dart b/moor_generator/lib/src/writer/tables/table_writer.dart index 74910861..35f62253 100644 --- a/moor_generator/lib/src/writer/tables/table_writer.dart +++ b/moor_generator/lib/src/writer/tables/table_writer.dart @@ -1,33 +1,37 @@ import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/state/session.dart'; import 'package:moor_generator/src/utils/string_escaper.dart'; -import 'package:moor_generator/src/writer/data_class_writer.dart'; -import 'package:moor_generator/src/writer/update_companion_writer.dart'; -import 'package:moor_generator/src/writer/memoized_getter.dart'; +import 'package:moor_generator/src/writer/tables/data_class_writer.dart'; +import 'package:moor_generator/src/writer/tables/update_companion_writer.dart'; +import 'package:moor_generator/src/writer/utils/memoized_getter.dart'; +import 'package:moor_generator/src/writer/writer.dart'; class TableWriter { final SpecifiedTable table; - final GeneratorSession session; + final Scope scope; - TableWriter(this.table, this.session); + StringBuffer _buffer; - void writeInto(StringBuffer buffer) { - writeDataClass(buffer); - writeTableInfoClass(buffer); + TableWriter(this.table, this.scope) { + _buffer = scope.leaf(); } - void writeDataClass(StringBuffer buffer) { - DataClassWriter(table, session).writeInto(buffer); - UpdateCompanionWriter(table, session).writeInto(buffer); + void writeInto() { + writeDataClass(); + writeTableInfoClass(); } - void writeTableInfoClass(StringBuffer buffer) { + void writeDataClass() { + DataClassWriter(table, scope.child()).write(); + UpdateCompanionWriter(table, scope.child()).write(); + } + + void writeTableInfoClass() { final dataClass = table.dartTypeName; final tableDslName = table.fromClass?.name ?? 'Table'; // class UsersTable extends Users implements TableInfo { - buffer + _buffer ..write('class ${table.tableInfoName} extends $tableDslName ' 'with TableInfo<${table.tableInfoName}, $dataClass> {\n') // should have a GeneratedDatabase reference that is set in the constructor @@ -37,15 +41,15 @@ class TableWriter { // Generate the columns for (var column in table.columns) { - _writeColumnVerificationMeta(buffer, column); - _writeColumnGetter(buffer, column); + _writeColumnVerificationMeta(column); + _writeColumnGetter(column); } // Generate $columns, $tableName, asDslTable getters final columnsWithGetters = table.columns.map((c) => c.dartGetterName).join(', '); - buffer + _buffer ..write( '@override\nList get \$columns => [$columnsWithGetters];\n') ..write('@override\n${table.tableInfoName} get asDslTable => this;\n') @@ -54,33 +58,33 @@ class TableWriter { ..write( '@override\nfinal String actualTableName = \'${table.sqlName}\';\n'); - _writeValidityCheckMethod(buffer); - _writePrimaryKeyOverride(buffer); + _writeValidityCheckMethod(); + _writePrimaryKeyOverride(); - _writeMappingMethod(buffer); - _writeReverseMappingMethod(buffer); + _writeMappingMethod(); + _writeReverseMappingMethod(); - _writeAliasGenerator(buffer); + _writeAliasGenerator(); - _writeConvertersAsStaticFields(buffer); - _overrideFieldsIfNeeded(buffer); + _writeConvertersAsStaticFields(); + _overrideFieldsIfNeeded(); // close class - buffer.write('}'); + _buffer.write('}'); } - void _writeConvertersAsStaticFields(StringBuffer buffer) { + void _writeConvertersAsStaticFields() { for (var converter in table.converters) { final typeName = converter.typeOfConverter.displayName; final code = converter.expression.toSource(); - buffer..write('static $typeName ${converter.fieldName} = $code;'); + _buffer..write('static $typeName ${converter.fieldName} = $code;'); } } - void _writeMappingMethod(StringBuffer buffer) { + void _writeMappingMethod() { final dataClassName = table.dartTypeName; - buffer + _buffer ..write( '@override\n$dataClassName map(Map data, {String tablePrefix}) {\n') ..write( @@ -90,15 +94,15 @@ class TableWriter { ..write('}\n'); } - void _writeReverseMappingMethod(StringBuffer buffer) { + void _writeReverseMappingMethod() { // Map entityToSql(covariant UpdateCompanion instance) - buffer + _buffer ..write('@override\nMap entityToSql(' '${table.updateCompanionName} d) {\n') ..write('final map = {};'); for (var column in table.columns) { - buffer.write('if (d.${column.dartGetterName}.present) {'); + _buffer.write('if (d.${column.dartGetterName}.present) {'); final mapSetter = 'map[${asDartLiteral(column.name.name)}] = ' 'Variable<${column.variableTypeName}, ${column.sqlTypeName}>'; @@ -106,26 +110,26 @@ class TableWriter { // apply type converter before writing the variable final converter = column.typeConverter; final fieldName = '${table.tableInfoName}.${converter.fieldName}'; - buffer + _buffer ..write('final converter = $fieldName;\n') ..write(mapSetter) ..write('(converter.mapToSql(d.${column.dartGetterName}.value));'); } else { // no type converter. Write variable directly - buffer + _buffer ..write(mapSetter) ..write('(') ..write('d.${column.dartGetterName}.value') ..write(');'); } - buffer.write('}'); + _buffer.write('}'); } - buffer.write('return map; \n}\n'); + _buffer.write('return map; \n}\n'); } - void _writeColumnGetter(StringBuffer buffer, SpecifiedColumn column) { + void _writeColumnGetter(SpecifiedColumn column) { final isNullable = column.nullable; final additionalParams = {}; final expressionBuffer = StringBuffer(); @@ -176,7 +180,7 @@ class TableWriter { expressionBuffer.write(');'); writeMemoizedGetterWithBody( - buffer: buffer, + buffer: _buffer, getterName: column.dartGetterName, returnType: column.implColumnTypeName, code: expressionBuffer.toString(), @@ -186,16 +190,15 @@ class TableWriter { ); } - void _writeColumnVerificationMeta( - StringBuffer buffer, SpecifiedColumn column) { + void _writeColumnVerificationMeta(SpecifiedColumn column) { // final VerificationMeta _targetDateMeta = const VerificationMeta('targetDate'); - buffer + _buffer ..write('final VerificationMeta ${_fieldNameForColumnMeta(column)} = ') ..write("const VerificationMeta('${column.dartGetterName}');\n"); } - void _writeValidityCheckMethod(StringBuffer buffer) { - buffer + void _writeValidityCheckMethod() { + _buffer ..write('@override\nVerificationContext validateIntegrity' '(${table.updateCompanionName} d, {bool isInserting = false}) {\n') ..write('final context = VerificationContext();\n'); @@ -207,13 +210,13 @@ class TableWriter { if (column.typeConverter != null) { // dont't verify custom columns, we assume that the user knows what // they're doing - buffer + _buffer ..write( 'context.handle($metaName, const VerificationResult.success());'); continue; } - buffer + _buffer ..write('if (d.$getterName.present) {\n') ..write('context.handle(' '$metaName, ' @@ -222,15 +225,15 @@ class TableWriter { ..write('context.missing($metaName);\n') ..write('}\n'); } - buffer.write('return context;\n}\n'); + _buffer.write('return context;\n}\n'); } String _fieldNameForColumnMeta(SpecifiedColumn column) { return '_${column.dartGetterName}Meta'; } - void _writePrimaryKeyOverride(StringBuffer buffer) { - buffer.write('@override\nSet get \$primaryKey => '); + void _writePrimaryKeyOverride() { + _buffer.write('@override\nSet get \$primaryKey => '); var primaryKey = table.primaryKey; // If there is an auto increment column, that forms the primary key. The @@ -239,37 +242,37 @@ class TableWriter { primaryKey ??= table.columns.where((c) => c.hasAI).toSet(); if (primaryKey.isEmpty) { - buffer.write('{};'); + _buffer.write('{};'); return; } - buffer.write('{'); + _buffer.write('{'); final pkList = primaryKey.toList(); for (var i = 0; i < pkList.length; i++) { final pk = pkList[i]; - buffer.write(pk.dartGetterName); + _buffer.write(pk.dartGetterName); if (i != pkList.length - 1) { - buffer.write(', '); + _buffer.write(', '); } } - buffer.write('};\n'); + _buffer.write('};\n'); } - void _writeAliasGenerator(StringBuffer buffer) { + void _writeAliasGenerator() { final typeName = table.tableInfoName; - buffer + _buffer ..write('@override\n') ..write('$typeName createAlias(String alias) {\n') ..write('return $typeName(_db, alias);') ..write('}'); } - void _overrideFieldsIfNeeded(StringBuffer buffer) { + void _overrideFieldsIfNeeded() { if (table.overrideWithoutRowId != null) { final value = table.overrideWithoutRowId ? 'true' : 'false'; - buffer + _buffer ..write('@override\n') ..write('final bool withoutRowId = $value;\n'); } @@ -278,14 +281,14 @@ class TableWriter { final value = table.overrideTableConstraints.map(asDartLiteral).join(', '); - buffer + _buffer ..write('@override\n') ..write('final List customConstraints = const [$value];\n'); } if (table.overrideDontWriteConstraints != null) { final value = table.overrideDontWriteConstraints ? 'true' : 'false'; - buffer + _buffer ..write('@override\n') ..write('final bool dontWriteConstraints = $value;\n'); } diff --git a/moor_generator/lib/src/writer/tables/update_companion_writer.dart b/moor_generator/lib/src/writer/tables/update_companion_writer.dart index 8f65f73c..7b81b69b 100644 --- a/moor_generator/lib/src/writer/tables/update_companion_writer.dart +++ b/moor_generator/lib/src/writer/tables/update_companion_writer.dart @@ -1,50 +1,54 @@ import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/state/session.dart'; +import 'package:moor_generator/src/writer/writer.dart'; class UpdateCompanionWriter { final SpecifiedTable table; - final GeneratorSession session; + final Scope scope; - UpdateCompanionWriter(this.table, this.session); + StringBuffer _buffer; - void writeInto(StringBuffer buffer) { - buffer.write('class ${table.updateCompanionName} ' - 'extends UpdateCompanion<${table.dartTypeName}> {\n'); - _writeFields(buffer); - _writeConstructor(buffer); - _writeInsertConstructor(buffer); - _writeCopyWith(buffer); - - buffer.write('}\n'); + UpdateCompanionWriter(this.table, this.scope) { + _buffer = scope.leaf(); } - void _writeFields(StringBuffer buffer) { + void write() { + _buffer.write('class ${table.updateCompanionName} ' + 'extends UpdateCompanion<${table.dartTypeName}> {\n'); + _writeFields(); + _writeConstructor(); + _writeInsertConstructor(); + _writeCopyWith(); + + _buffer.write('}\n'); + } + + void _writeFields() { for (var column in table.columns) { - buffer.write('final Value<${column.dartTypeName}>' + _buffer.write('final Value<${column.dartTypeName}>' ' ${column.dartGetterName};\n'); } } - void _writeConstructor(StringBuffer buffer) { - buffer.write('const ${table.updateCompanionName}({'); + void _writeConstructor() { + _buffer.write('const ${table.updateCompanionName}({'); for (var column in table.columns) { - buffer.write('this.${column.dartGetterName} = const Value.absent(),'); + _buffer.write('this.${column.dartGetterName} = const Value.absent(),'); } - buffer.write('});\n'); + _buffer.write('});\n'); } /// Writes a special `.insert` constructor. All columns which may not be /// absent during insert are marked `@required` here. Also, we don't need to /// use value wrappers here - `Value.absent` simply isn't an option. - void _writeInsertConstructor(StringBuffer buffer) { + void _writeInsertConstructor() { final requiredColumns = {}; // can't be constant because we use initializers (this.a = Value(a)). // for a parameter a which is only potentially constant. - buffer.write('${table.updateCompanionName}.insert({'); + _buffer.write('${table.updateCompanionName}.insert({'); // Say we had two required columns a and c, and an optional column b. // .insert({ @@ -59,47 +63,47 @@ class UpdateCompanionWriter { if (column.requiredDuringInsert) { requiredColumns.add(column); - buffer.write('@required ${column.dartTypeName} $param,'); + _buffer.write('@required ${column.dartTypeName} $param,'); } else { - buffer.write('this.$param = const Value.absent(),'); + _buffer.write('this.$param = const Value.absent(),'); } } - buffer.write('})'); + _buffer.write('})'); var first = true; for (var required in requiredColumns) { if (first) { - buffer.write(': '); + _buffer.write(': '); first = false; } else { - buffer.write(', '); + _buffer.write(', '); } final param = required.dartGetterName; - buffer.write('$param = Value($param)'); + _buffer.write('$param = Value($param)'); } - buffer.write(';\n'); + _buffer.write(';\n'); } - void _writeCopyWith(StringBuffer buffer) { - buffer.write('${table.updateCompanionName} copyWith({'); + void _writeCopyWith() { + _buffer.write('${table.updateCompanionName} copyWith({'); var first = true; for (var column in table.columns) { if (!first) { - buffer.write(', '); + _buffer.write(', '); } first = false; - buffer.write('Value<${column.dartTypeName}> ${column.dartGetterName}'); + _buffer.write('Value<${column.dartTypeName}> ${column.dartGetterName}'); } - buffer + _buffer ..write('}) {\n') // ..write('return ${table.updateCompanionName}('); for (var column in table.columns) { final name = column.dartGetterName; - buffer.write('$name: $name ?? this.$name,'); + _buffer.write('$name: $name ?? this.$name,'); } - buffer.write(');\n}\n'); + _buffer.write(');\n}\n'); } } diff --git a/moor_generator/pubspec.yaml b/moor_generator/pubspec.yaml index 97f2bbef..44769e84 100644 --- a/moor_generator/pubspec.yaml +++ b/moor_generator/pubspec.yaml @@ -18,6 +18,7 @@ dependencies: source_gen: ^0.9.4 source_span: ^1.5.5 build: ^1.1.0 + logging: '>=0.11.0 <1.0.0' build_config: '>=0.3.1 <1.0.0' moor: ^1.7.1 meta: ^1.1.0 diff --git a/moor_generator/test/analyzer/dart/dart_test.dart b/moor_generator/test/analyzer/dart/dart_test.dart index e77a47cd..b49de7ca 100644 --- a/moor_generator/test/analyzer/dart/dart_test.dart +++ b/moor_generator/test/analyzer/dart/dart_test.dart @@ -1,5 +1,6 @@ import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; +import 'package:build/build.dart'; import 'package:moor_generator/src/analyzer/dart/parser.dart'; import 'package:test/test.dart'; @@ -8,7 +9,7 @@ import '../../utils/test_backend.dart'; void main() { test('return expression of methods', () async { final backend = TestBackend({ - 'test_lib|main.dart': r''' + AssetId.parse('test_lib|lib/main.dart'): r''' class Test { String get getter => 'foo'; String function() => 'bar'; @@ -19,7 +20,8 @@ void main() { ''' }); - final backendTask = backend.startTask('test_lib|main.dart'); + final backendTask = + backend.startTask(Uri.parse('package:test_lib/main.dart')); final dartTask = await backend.session.startDartTask(backendTask); final parser = MoorDartParser(dartTask); diff --git a/moor_generator/test/analyzer/dart/table_parser_test.dart b/moor_generator/test/analyzer/dart/table_parser_test.dart index bc6fa258..dea23597 100644 --- a/moor_generator/test/analyzer/dart/table_parser_test.dart +++ b/moor_generator/test/analyzer/dart/table_parser_test.dart @@ -1,3 +1,4 @@ +import 'package:build/build.dart'; import 'package:moor_generator/src/analyzer/dart/parser.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/model/specified_column.dart'; @@ -12,7 +13,7 @@ void main() { MoorDartParser parser; setUpAll(() { backend = TestBackend({ - 'test_lib|main.dart': r''' + AssetId.parse('test_lib|lib/main.dart'): r''' import 'package:moor/moor.dart'; class TableWithCustomName extends Table { @@ -47,7 +48,7 @@ void main() { }); setUp(() async { - final task = backend.startTask('test_lib|main.dart'); + final task = backend.startTask(Uri.parse('package:test_lib/main.dart')); dartTask = await backend.session.startDartTask(task); parser = MoorDartParser(dartTask); }); diff --git a/moor_generator/test/parser/moor/moor_parser_test.dart b/moor_generator/test/parser/moor/moor_parser_test.dart index 375622a4..55242a0e 100644 --- a/moor_generator/test/parser/moor/moor_parser_test.dart +++ b/moor_generator/test/parser/moor/moor_parser_test.dart @@ -1,5 +1,5 @@ -import 'package:moor_generator/src/parser/moor/moor_analyzer.dart'; -import 'package:moor_generator/src/parser/sql/type_mapping.dart'; +import 'package:moor_generator/src/analyzer/moor/parser.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; import 'package:test_api/test_api.dart'; void main() { @@ -11,13 +11,13 @@ CREATE TABLE users( '''; test('extracts table structure from .moor files', () async { - final analyzer = MoorAnalyzer(content); - final result = await analyzer.analyze(); + final task = MoorTask(null, null, content); + final analyzer = MoorParser(task); + final result = await analyzer.parseAndAnalyze(); - expect(result.errors, isEmpty); + expect(task.errors.errors, isEmpty); - final table = - result.parsedFile.declaredTables.single.extractTable(TypeMapper()); + final table = result.declaredTables.single; expect(table.sqlName, 'users'); }); diff --git a/moor_generator/test/parser/parser_test.dart b/moor_generator/test/parser/parser_test.dart deleted file mode 100644 index 02ff262a..00000000 --- a/moor_generator/test/parser/parser_test.dart +++ /dev/null @@ -1,144 +0,0 @@ -import 'package:analyzer/dart/element/element.dart'; -import 'package:moor_generator/src/model/specified_column.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/state/generator_state.dart'; -import 'package:moor_generator/src/state/options.dart'; -import 'package:moor_generator/src/parser/table_parser.dart'; -import 'package:moor_generator/src/state/session.dart'; -import 'package:test_api/test_api.dart'; -import 'package:build_test/build_test.dart'; - -void main() async { - LibraryElement testLib; - GeneratorState state; - GeneratorSession session; - - setUpAll(() async { - testLib = await resolveSource(r''' - library test_parser; - - import 'package:moor/moor.dart'; - - class TableWithCustomName extends Table { - @override - String get tableName => "my-fancy-table" - } - - class Users extends Table { - IntColumn get id => integer().autoIncrement()(); - TextColumn get name => text().named("user_name").withLength(min: 6, max: 32)(); - TextColumn get onlyMax => text().withLength(max: 100)(); - - DateTimeColumn get defaults => dateTime().withDefault(currentDate)(); - } - - class CustomPrimaryKey extends Table { - IntColumn get partA => integer()(); - IntColumn get partB => integer().customConstraint('custom')(); - - @override - Set get primaryKey => {partA, partB}; - } - - class WrongName extends Table { - - String constructTableName() { - return "my-table-name"; - } - - @override - String get tableName => constructTableName(); - } - ''', (r) => r.findLibraryByName('test_parser')); - }); - - setUp(() { - state = useState(() => GeneratorState(const MoorOptions.defaults())); - session = state.startSession(null); - }); - - Future parse(String name) { - return TableParser(session).parse(testLib.getType(name)); - } - - group('SQL table name', () { - test('should parse correctly when valid', () async { - final parsed = await parse('TableWithCustomName'); - expect(parsed.sqlName, equals('my-fancy-table')); - }); - - test('should use class name if table name is not specified', () async { - final parsed = await parse('Users'); - expect(parsed.sqlName, equals('users')); - }); - - test('should not parse for complex methods', () async { - await TableParser(session).parse(testLib.getType('WrongName')); - - expect(session.errors.errors, isNotEmpty); - }); - }); - - group('Columns', () { - test('should use field name if no name has been set explicitely', () async { - final table = await parse('Users'); - final idColumn = - table.columns.singleWhere((col) => col.name.name == 'id'); - - expect(idColumn.name, equals(ColumnName.implicitly('id'))); - }); - - test('should use explicit name, if it exists', () async { - final table = await parse('Users'); - final idColumn = - table.columns.singleWhere((col) => col.name.name == 'user_name'); - - expect(idColumn.name, equals(ColumnName.explicitly('user_name'))); - }); - - test('should parse min and max length for text columns', () async { - final table = await parse('Users'); - final idColumn = - table.columns.singleWhere((col) => col.name.name == 'user_name'); - - expect(idColumn.features, - contains(LimitingTextLength.withLength(min: 6, max: 32))); - }); - - test('should only parse max length when relevant', () async { - final table = await parse('Users'); - final idColumn = - table.columns.singleWhere((col) => col.dartGetterName == 'onlyMax'); - - expect( - idColumn.features, contains(LimitingTextLength.withLength(max: 100))); - }); - - test('parses custom constraints', () async { - final table = await parse('CustomPrimaryKey'); - - final partA = - table.columns.singleWhere((c) => c.dartGetterName == 'partA'); - final partB = - table.columns.singleWhere((c) => c.dartGetterName == 'partB'); - - expect(partB.customConstraints, 'custom'); - expect(partA.customConstraints, isNull); - }); - - test('parsed default values', () async { - final table = await parse('Users'); - final defaultsColumn = - table.columns.singleWhere((c) => c.name.name == 'defaults'); - - expect(defaultsColumn.defaultArgument.toString(), 'currentDate'); - }); - }); - - test('parses custom primary keys', () async { - final table = await parse('CustomPrimaryKey'); - - expect(table.primaryKey, containsAll(table.columns)); - expect(table.columns.any((column) => column.hasAI), isFalse); - }); -} diff --git a/moor_generator/test/parser/sql/query_handler_test.dart b/moor_generator/test/parser/sql/query_handler_test.dart index 23a3ef20..d7f9ab0b 100644 --- a/moor_generator/test/parser/sql/query_handler_test.dart +++ b/moor_generator/test/parser/sql/query_handler_test.dart @@ -1,5 +1,5 @@ -import 'package:moor_generator/src/parser/sql/query_handler.dart'; -import 'package:moor_generator/src/parser/sql/type_mapping.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/query_handler.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; import 'package:sqlparser/sqlparser.dart'; import 'package:test/test.dart'; diff --git a/moor_generator/test/utils/test_backend.dart b/moor_generator/test/utils/test_backend.dart index a08eebb9..5a6ada24 100644 --- a/moor_generator/test/utils/test_backend.dart +++ b/moor_generator/test/utils/test_backend.dart @@ -3,10 +3,11 @@ import 'dart:async'; import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart'; import 'package:build_test/build_test.dart'; +import 'package:logging/logging.dart'; import 'package:moor_generator/src/backends/backend.dart'; class TestBackend extends Backend { - final Map fakeContent; + final Map fakeContent; Resolver _resolver; final Completer _initCompleter = Completer(); @@ -21,15 +22,15 @@ class TestBackend extends Backend { } void _init() { - resolveSources(fakeContent, (r) { + resolveSources(fakeContent.map((k, v) => MapEntry(k.toString(), v)), (r) { _resolver = r; _initCompleter.complete(); return _finish.future; }); } - BackendTask startTask(String path) { - return _TestBackendTask(this, path); + BackendTask startTask(Uri uri) { + return _TestBackendTask(this, uri); } void finish() { @@ -41,19 +42,22 @@ class _TestBackendTask extends BackendTask { final TestBackend backend; @override - final String entrypoint; + final Uri entrypoint; + + @override + Logger get log => null; _TestBackendTask(this.backend, this.entrypoint); @override - Future readMoor(String path) async { + Future readMoor(Uri path) async { await backend._ready; - return backend.fakeContent[path]; + return backend.fakeContent[AssetId.resolve(path.toString())]; } @override - Future resolveDart(String path) async { + Future resolveDart(Uri path) async { await backend._ready; - return await backend._resolver.libraryFor(AssetId.parse(path)); + return await backend._resolver.libraryFor(AssetId.resolve(path.toString())); } } diff --git a/moor_generator/test/parser/writer/utils/hash_code_test.dart b/moor_generator/test/writer/utils/hash_code_test.dart similarity index 100% rename from moor_generator/test/parser/writer/utils/hash_code_test.dart rename to moor_generator/test/writer/utils/hash_code_test.dart From 35fcdb2c0f73689d3743a575fdd97ee129e1dd2f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Sep 2019 19:30:05 +0200 Subject: [PATCH 045/117] Implement error handling in the refactored builder --- moor/example/example.g.dart | 18 +++---- moor/test/data/tables/todos.g.dart | 52 +++++++++---------- moor_generator/lib/src/analyzer/errors.dart | 31 +++++++++++ moor_generator/lib/src/analyzer/session.dart | 25 ++++----- .../src/analyzer/sql_queries/sql_parser.dart | 6 +-- .../lib/src/writer/queries/query_writer.dart | 6 +-- .../lib/src/writer/tables/table_writer.dart | 6 +-- moor_generator/lib/src/writer/writer.dart | 6 ++- 8 files changed, 92 insertions(+), 58 deletions(-) diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index 7b81c753..68b79aec 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -829,15 +829,6 @@ class $IngredientInRecipesTable extends IngredientInRecipes } } -class TotalWeightResult { - final String title; - final int totalWeight; - TotalWeightResult({ - this.title, - this.totalWeight, - }); -} - abstract class _$Database extends GeneratedDatabase { _$Database(QueryExecutor e) : super(const SqlTypeSystem.withDefaults(), e); $CategoriesTable _categories; @@ -879,3 +870,12 @@ abstract class _$Database extends GeneratedDatabase { List get allTables => [categories, recipes, ingredients, ingredientInRecipes]; } + +class TotalWeightResult { + final String title; + final int totalWeight; + TotalWeightResult({ + this.title, + this.totalWeight, + }); +} diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 74b0e7e2..d0bcdf53 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1282,32 +1282,6 @@ class $PureDefaultsTable extends PureDefaults } } -class AllTodosWithCategoryResult { - final int id; - final String title; - final String content; - final DateTime targetDate; - final int category; - final int catId; - final String catDesc; - AllTodosWithCategoryResult({ - this.id, - this.title, - this.content, - this.targetDate, - this.category, - this.catId, - this.catDesc, - }); -} - -class FindCustomResult { - final MyCustomObject custom; - FindCustomResult({ - this.custom, - }); -} - abstract class _$TodoDb extends GeneratedDatabase { _$TodoDb(QueryExecutor e) : super(const SqlTypeSystem.withDefaults(), e); $TodosTableTable _todosTable; @@ -1480,6 +1454,32 @@ abstract class _$TodoDb extends GeneratedDatabase { ]; } +class AllTodosWithCategoryResult { + final int id; + final String title; + final String content; + final DateTime targetDate; + final int category; + final int catId; + final String catDesc; + AllTodosWithCategoryResult({ + this.id, + this.title, + this.content, + this.targetDate, + this.category, + this.catId, + this.catDesc, + }); +} + +class FindCustomResult { + final MyCustomObject custom; + FindCustomResult({ + this.custom, + }); +} + // ************************************************************************** // DaoGenerator // ************************************************************************** diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index 8673eaf6..35998536 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -1,14 +1,30 @@ import 'package:analyzer/dart/element/element.dart'; import 'package:collection/collection.dart'; import 'package:meta/meta.dart'; +import 'package:source_gen/source_gen.dart'; import 'package:source_span/source_span.dart'; +typedef LogFunction = void Function(dynamic message, + [Object error, StackTrace stackTrace]); + /// Base class for errors that can be presented to an user. class MoorError { final Severity severity; final String message; MoorError({@required this.severity, this.message}); + + bool get isError => + severity == Severity.criticalError || severity == Severity.error; + + @override + String toString() { + return 'Error: $message'; + } + + void writeDescription(LogFunction log) { + log(message); + } } class ErrorInDartCode extends MoorError { @@ -19,6 +35,16 @@ class ErrorInDartCode extends MoorError { this.affectedElement, Severity severity = Severity.warning}) : super(severity: severity, message: message); + + @override + void writeDescription(LogFunction log) { + if (affectedElement != null) { + final span = spanForElement(affectedElement); + log(span.message(message)); + } else { + log(message); + } + } } class ErrorInMoorFile extends MoorError { @@ -29,6 +55,11 @@ class ErrorInMoorFile extends MoorError { String message, Severity severity = Severity.warning}) : super(message: message, severity: severity); + + @override + void writeDescription(LogFunction log) { + log(span.message(message)); + } } class ErrorSink { diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 7831bfdf..24812d28 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -48,20 +48,18 @@ abstract class FileTask { FutureOr compute(); void printErrors() { - /* - * if (session.errors.errors.isNotEmpty) { - print('Warning: There were some errors while running ' - 'moor_generator on ${buildStep.inputId.path}:'); + final foundErrors = errors.errors; + if (foundErrors.isNotEmpty) { + final log = backendTask.log; - for (var error in session.errors.errors) { - print(error.message); + log.warning('There were some errors while running ' + 'moor_generator on ${backendTask.entrypoint}:'); - if (error.affectedElement != null) { - final span = spanForElement(error.affectedElement); - print('${span.start.toolString}\n${span.highlight()}'); - } + for (var error in foundErrors) { + final printer = error.isError ? log.warning : log.info; + error.writeDescription(printer); } - } */ + } } } @@ -116,7 +114,10 @@ class DartTask extends FileTask { } else { return parser.parseTable(type.element as ClassElement); } - })).then((list) => List.from(list)); // make growable + })).then((list) { + // only keep tables that were resolved successfully + return List.from(list.where((t) => t != null)); + }); } /// Reads all tables declared in sql by a `.moor` file in [paths]. diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index 2fc74859..60dd2219 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -38,21 +38,21 @@ class SqlParser { } catch (e, s) { task.reportError(MoorError( severity: Severity.criticalError, - message: 'Error while trying to parse $sql: $e, $s')); + message: 'Error while trying to parse $key: $e, $s')); return; } for (var error in context.errors) { task.reportError(MoorError( severity: Severity.warning, - message: 'The sql query $sql is invalid: $error', + message: 'The sql query $key is invalid: $error', )); } try { foundQueries.add(QueryHandler(name, context, _mapper).handle()); } catch (e, s) { - log.warning('Error while generating APIs for ${context.sql}', e, s); + log.warning('Error while generating APIs for $key', e, s); } }); } diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 078ffbd4..e6233fd4 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -311,15 +311,15 @@ class QueryWriter { // write everything that comes before this var into the_buffer final currentIndex = sqlVar.firstPosition; final queryPart = query.sql.substring(lastIndex, currentIndex); - _buffer.write(escapeForDart(queryPart)); + buffer.write(escapeForDart(queryPart)); lastIndex = sqlVar.lastPosition; // write the ($expandedVar) par - _buffer.write('(\$${_expandedName(moorVar)})'); + buffer.write('(\$${_expandedName(moorVar)})'); } // write the final part after the last variable, plus the ending ' - _buffer..write(escapeForDart(query.sql.substring(lastIndex)))..write("'"); + buffer..write(escapeForDart(query.sql.substring(lastIndex)))..write("'"); return buffer.toString(); } diff --git a/moor_generator/lib/src/writer/tables/table_writer.dart b/moor_generator/lib/src/writer/tables/table_writer.dart index 35f62253..65438582 100644 --- a/moor_generator/lib/src/writer/tables/table_writer.dart +++ b/moor_generator/lib/src/writer/tables/table_writer.dart @@ -12,9 +12,7 @@ class TableWriter { StringBuffer _buffer; - TableWriter(this.table, this.scope) { - _buffer = scope.leaf(); - } + TableWriter(this.table, this.scope); void writeInto() { writeDataClass(); @@ -27,6 +25,8 @@ class TableWriter { } void writeTableInfoClass() { + _buffer = scope.leaf(); + final dataClass = table.dartTypeName; final tableDslName = table.fromClass?.name ?? 'Table'; diff --git a/moor_generator/lib/src/writer/writer.dart b/moor_generator/lib/src/writer/writer.dart index 1fad3f04..54773ff2 100644 --- a/moor_generator/lib/src/writer/writer.dart +++ b/moor_generator/lib/src/writer/writer.dart @@ -10,10 +10,12 @@ import 'package:moor_generator/src/backends/build/moor_builder.dart'; /// [StringBuffer] to the generators that will get ugly to manage, but when /// passing a [Scope] we will always be able to write code in a parent scope. class Writer { - final Scope _root = Scope(parent: null); + /* late final */ Scope _root; final MoorOptions options; - Writer(this.options); + Writer(this.options) { + _root = Scope(parent: null, writer: this); + } String writeGenerated() => _leafNodes(_root).join(); From ba772ef07fd6d7a1edfdc09d9003d0d9048053e8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 3 Sep 2019 21:24:59 +0200 Subject: [PATCH 046/117] Better error handling when parsing multiple sql statements --- moor_generator/lib/src/analyzer/session.dart | 9 +++- sqlparser/lib/src/reader/parser/parser.dart | 16 +++++-- .../test/parser/multiple_statements.dart | 48 +++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) create mode 100644 sqlparser/test/parser/multiple_statements.dart diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 24812d28..79d2a42e 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -124,7 +124,14 @@ class DartTask extends FileTask { Future> resolveIncludes(Iterable paths) { return Stream.fromFutures(paths.map( (path) => session.startMoorTask(backendTask, uri: Uri.parse(path)))) - .asyncMap((task) => task.compute()) + .asyncMap((task) async { + final result = await task.compute(); + + // add errors from nested task to this task as well. + task.errors.errors.forEach(reportError); + + return result; + }) .expand((file) => file.declaredTables) .toList(); } diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index a47fbc7f..bc1053e1 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -151,9 +151,6 @@ abstract class ParserBase { WindowDefinition _windowDefinition(); } -// todo better error handling and synchronisation, like it's done here: -// https://craftinginterpreters.com/parsing-expressions.html#synchronizing-a-recursive-descent-parser - class Parser extends ParserBase with ExpressionParser, SchemaParser, CrudParser { Parser(List tokens, {bool useMoor = false}) : super(tokens, useMoor); @@ -180,8 +177,19 @@ class Parser extends ParserBase List statements() { final stmts = []; while (!_isAtEnd) { - stmts.add(statement(expectEnd: false)); + try { + stmts.add(statement(expectEnd: false)); + } on ParsingError catch (_) { + // the error is added to the list errors, so ignore. We skip to the next + // semicolon to parse the next statement. + _synchronize(); + } } return stmts; } + + void _synchronize() { + // fast-forward to the token after th next semicolon + while (!_isAtEnd && _advance().type != TokenType.semicolon) {} + } } diff --git a/sqlparser/test/parser/multiple_statements.dart b/sqlparser/test/parser/multiple_statements.dart new file mode 100644 index 00000000..507e0aa5 --- /dev/null +++ b/sqlparser/test/parser/multiple_statements.dart @@ -0,0 +1,48 @@ +import 'package:sqlparser/sqlparser.dart'; +import 'package:sqlparser/src/reader/parser/parser.dart'; +import 'package:sqlparser/src/reader/tokenizer/scanner.dart'; +import 'package:sqlparser/src/utils/ast_equality.dart'; +import 'package:test/test.dart'; + +void main() { + test('can parse multiple statements', () { + final sql = 'UPDATE tbl SET a = b; SELECT * FROM tbl;'; + final tokens = Scanner(sql).scanTokens(); + final statements = Parser(tokens).statements(); + + enforceEqual( + statements[0], + UpdateStatement( + table: TableReference('tbl', null), + set: [ + SetComponent( + column: Reference(columnName: 'a'), + expression: Reference(columnName: 'b'), + ), + ], + ), + ); + enforceEqual( + statements[1], + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + ), + ); + }); + + test('recovers from invalid statements', () { + final sql = 'UPDATE tbl SET a = * d; SELECT * FROM tbl;'; + final tokens = Scanner(sql).scanTokens(); + final statements = Parser(tokens).statements(); + + expect(statements, hasLength(1)); + enforceEqual( + statements[0], + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + ), + ); + }); +} From 1bbc26767e6c722456aa93687918e8d923131e62 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 4 Sep 2019 16:13:37 +0200 Subject: [PATCH 047/117] Assert that the database is not opened multiple times. --- .../src/runtime/executor/helpers/engines.dart | 31 +++------ moor_flutter/lib/moor_flutter.dart | 63 +++++++++++-------- moor_flutter/pubspec.lock | 12 ++-- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/moor/lib/src/runtime/executor/helpers/engines.dart b/moor/lib/src/runtime/executor/helpers/engines.dart index c6a2c70b..88102171 100644 --- a/moor/lib/src/runtime/executor/helpers/engines.dart +++ b/moor/lib/src/runtime/executor/helpers/engines.dart @@ -220,7 +220,6 @@ class _BeforeOpeningExecutor extends QueryExecutor /// work to a [DatabaseDelegate]. class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { final DatabaseDelegate delegate; - Completer _openingCompleter; @override bool logStatements; @@ -240,29 +239,15 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { } @override - Future ensureOpen() async { - // if we're already opening the database or if its already open, return that - // status - if (_openingCompleter != null) { - return _openingCompleter.future; - } + Future ensureOpen() { + return _synchronized(() async { + final alreadyOpen = await delegate.isOpen; + if (alreadyOpen) return true; - final alreadyOpen = await delegate.isOpen; - if (alreadyOpen) return true; - - // ignore: invariant_booleans - if (_openingCompleter != null) { - return _openingCompleter.future; - } - - // not already open or opening. Open the database now! - _openingCompleter = Completer(); - await delegate.open(databaseInfo); - await _runMigrations(); - - _openingCompleter.complete(true); - _openingCompleter = null; - return true; + await delegate.open(databaseInfo); + await _runMigrations(); + return true; + }); } Future _runMigrations() async { diff --git a/moor_flutter/lib/moor_flutter.dart b/moor_flutter/lib/moor_flutter.dart index 8ba481bf..f4a2e74c 100644 --- a/moor_flutter/lib/moor_flutter.dart +++ b/moor_flutter/lib/moor_flutter.dart @@ -20,6 +20,9 @@ export 'package:moor/moor.dart'; /// doesn't exist. typedef DatabaseCreator = FutureOr Function(File file); +// todo remove: Left in for debugging purposes +var _debugIsOpening = false; + class _SqfliteDelegate extends DatabaseDelegate with _SqfliteExecutor { int _loadedSchemaVersion; @override @@ -50,32 +53,42 @@ class _SqfliteDelegate extends DatabaseDelegate with _SqfliteExecutor { @override Future open([GeneratedDatabase db]) async { - String resolvedPath; - if (inDbFolder) { - resolvedPath = join(await s.getDatabasesPath(), path); - } else { - resolvedPath = path; + assert( + !_debugIsOpening && !isOpen, + 'Database opened multiple times, this should never happen. ' + 'Please report this at https://github.com/simolus3/moor/issues/135 to help fix this!'); + _debugIsOpening = true; + + try { + String resolvedPath; + if (inDbFolder) { + resolvedPath = join(await s.getDatabasesPath(), path); + } else { + resolvedPath = path; + } + + final file = File(resolvedPath); + if (creator != null && !await file.exists()) { + await creator(file); + } + + // default value when no migration happened + _loadedSchemaVersion = db.schemaVersion; + + this.db = await s.openDatabase( + resolvedPath, + version: db.schemaVersion, + onCreate: (db, version) { + _loadedSchemaVersion = 0; + }, + onUpgrade: (db, from, to) { + _loadedSchemaVersion = from; + }, + singleInstance: singleInstance, + ); + } finally { + _debugIsOpening = false; } - - final file = File(resolvedPath); - if (creator != null && !await file.exists()) { - await creator(file); - } - - // default value when no migration happened - _loadedSchemaVersion = db.schemaVersion; - - this.db = await s.openDatabase( - resolvedPath, - version: db.schemaVersion, - onCreate: (db, version) { - _loadedSchemaVersion = 0; - }, - onUpgrade: (db, from, to) { - _loadedSchemaVersion = from; - }, - singleInstance: singleInstance, - ); } @override diff --git a/moor_flutter/pubspec.lock b/moor_flutter/pubspec.lock index ab285a18..5837c258 100644 --- a/moor_flutter/pubspec.lock +++ b/moor_flutter/pubspec.lock @@ -7,14 +7,14 @@ packages: name: async url: "https://pub.dartlang.org" source: hosted - version: "2.3.0" + version: "2.2.0" boolean_selector: dependency: transitive description: name: boolean_selector url: "https://pub.dartlang.org" source: hosted - version: "1.0.5" + version: "1.0.4" charcode: dependency: transitive description: @@ -52,14 +52,14 @@ packages: name: meta url: "https://pub.dartlang.org" source: hosted - version: "1.1.7" + version: "1.1.6" moor: dependency: "direct main" description: path: "../moor" relative: true source: path - version: "1.6.0" + version: "1.7.1" path: dependency: "direct main" description: @@ -73,7 +73,7 @@ packages: name: pedantic url: "https://pub.dartlang.org" source: hosted - version: "1.8.0+1" + version: "1.7.0" quiver: dependency: transitive description: @@ -99,7 +99,7 @@ packages: name: sqflite url: "https://pub.dartlang.org" source: hosted - version: "1.1.6+3" + version: "1.1.6+4" stack_trace: dependency: transitive description: From 3c56e0ad6e2598a5c394213c92a65273b7420a1c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 4 Sep 2019 18:38:55 +0200 Subject: [PATCH 048/117] Fix race condition in database opening --- extras/integration_tests/flutter_db/pubspec.lock | 6 +++--- moor/lib/src/runtime/executor/helpers/engines.dart | 4 +++- moor_flutter/lib/moor_flutter.dart | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/extras/integration_tests/flutter_db/pubspec.lock b/extras/integration_tests/flutter_db/pubspec.lock index 0afd92e4..2b92d0f6 100644 --- a/extras/integration_tests/flutter_db/pubspec.lock +++ b/extras/integration_tests/flutter_db/pubspec.lock @@ -159,7 +159,7 @@ packages: name: meta url: "https://pub.dartlang.org" source: hosted - version: "1.1.7" + version: "1.1.6" mime: dependency: transitive description: @@ -173,14 +173,14 @@ packages: path: "../../../moor" relative: true source: path - version: "1.6.0" + version: "1.7.1" moor_flutter: dependency: "direct main" description: path: "../../../moor_flutter" relative: true source: path - version: "1.6.0" + version: "1.7.0" multi_server_socket: dependency: transitive description: diff --git a/moor/lib/src/runtime/executor/helpers/engines.dart b/moor/lib/src/runtime/executor/helpers/engines.dart index 88102171..105e3171 100644 --- a/moor/lib/src/runtime/executor/helpers/engines.dart +++ b/moor/lib/src/runtime/executor/helpers/engines.dart @@ -232,6 +232,8 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { @override SqlDialect get dialect => delegate.dialect; + final Lock _openingLock = Lock(); + DelegatedDatabase(this.delegate, {this.logStatements, this.isSequential = false}) { // not using default value because it's commonly set to null @@ -240,7 +242,7 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { @override Future ensureOpen() { - return _synchronized(() async { + return _openingLock.synchronized(() async { final alreadyOpen = await delegate.isOpen; if (alreadyOpen) return true; diff --git a/moor_flutter/lib/moor_flutter.dart b/moor_flutter/lib/moor_flutter.dart index f4a2e74c..0965521e 100644 --- a/moor_flutter/lib/moor_flutter.dart +++ b/moor_flutter/lib/moor_flutter.dart @@ -53,6 +53,7 @@ class _SqfliteDelegate extends DatabaseDelegate with _SqfliteExecutor { @override Future open([GeneratedDatabase db]) async { + print('uses fix for the opening problem.'); assert( !_debugIsOpening && !isOpen, 'Database opened multiple times, this should never happen. ' From a17ad135426d0cb51fc349963e99cad09685cb6f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 4 Sep 2019 21:20:09 +0200 Subject: [PATCH 049/117] Report more errors when database is opened --- .../src/runtime/executor/helpers/engines.dart | 18 +++++++++++++++--- moor_flutter/lib/moor_flutter.dart | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/moor/lib/src/runtime/executor/helpers/engines.dart b/moor/lib/src/runtime/executor/helpers/engines.dart index 105e3171..bfa56029 100644 --- a/moor/lib/src/runtime/executor/helpers/engines.dart +++ b/moor/lib/src/runtime/executor/helpers/engines.dart @@ -240,14 +240,26 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { logStatements ??= false; } + var _ensureOpenCalledCounter = 0; // todo remove + @override Future ensureOpen() { - return _openingLock.synchronized(() async { - final alreadyOpen = await delegate.isOpen; - if (alreadyOpen) return true; + final call = ++_ensureOpenCalledCounter; + print('ensure_open_$call: called'); + return _openingLock.synchronized(() async { + print('ensure_open_$call: lock aquired'); + final alreadyOpen = await delegate.isOpen; + if (alreadyOpen) { + print('ensure_open_$call: was already open'); + return true; + } + + print('ensure_open_$call: opening database'); await delegate.open(databaseInfo); + print('ensure_open_$call: running migrations'); await _runMigrations(); + print('ensure_open_$call: done with opening'); return true; }); } diff --git a/moor_flutter/lib/moor_flutter.dart b/moor_flutter/lib/moor_flutter.dart index 0965521e..8271e22f 100644 --- a/moor_flutter/lib/moor_flutter.dart +++ b/moor_flutter/lib/moor_flutter.dart @@ -53,11 +53,11 @@ class _SqfliteDelegate extends DatabaseDelegate with _SqfliteExecutor { @override Future open([GeneratedDatabase db]) async { - print('uses fix for the opening problem.'); assert( !_debugIsOpening && !isOpen, 'Database opened multiple times, this should never happen. ' - 'Please report this at https://github.com/simolus3/moor/issues/135 to help fix this!'); + 'Please report this at https://github.com/simolus3/moor/issues/135 to help fix this!' + 'Db already opened: $isOpen. Other operation in progress: $_debugIsOpening'); _debugIsOpening = true; try { From 81986d2010256d52f0b9d1f95356fd209b0c9ad8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 4 Sep 2019 22:11:48 +0200 Subject: [PATCH 050/117] Remove debug logs used to fix the opening problem --- .../src/runtime/executor/helpers/engines.dart | 10 --- moor_flutter/lib/moor_flutter.dart | 64 ++++++++----------- 2 files changed, 25 insertions(+), 49 deletions(-) diff --git a/moor/lib/src/runtime/executor/helpers/engines.dart b/moor/lib/src/runtime/executor/helpers/engines.dart index bfa56029..d89df199 100644 --- a/moor/lib/src/runtime/executor/helpers/engines.dart +++ b/moor/lib/src/runtime/executor/helpers/engines.dart @@ -240,26 +240,16 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { logStatements ??= false; } - var _ensureOpenCalledCounter = 0; // todo remove - @override Future ensureOpen() { - final call = ++_ensureOpenCalledCounter; - print('ensure_open_$call: called'); - return _openingLock.synchronized(() async { - print('ensure_open_$call: lock aquired'); final alreadyOpen = await delegate.isOpen; if (alreadyOpen) { - print('ensure_open_$call: was already open'); return true; } - print('ensure_open_$call: opening database'); await delegate.open(databaseInfo); - print('ensure_open_$call: running migrations'); await _runMigrations(); - print('ensure_open_$call: done with opening'); return true; }); } diff --git a/moor_flutter/lib/moor_flutter.dart b/moor_flutter/lib/moor_flutter.dart index 8271e22f..8ba481bf 100644 --- a/moor_flutter/lib/moor_flutter.dart +++ b/moor_flutter/lib/moor_flutter.dart @@ -20,9 +20,6 @@ export 'package:moor/moor.dart'; /// doesn't exist. typedef DatabaseCreator = FutureOr Function(File file); -// todo remove: Left in for debugging purposes -var _debugIsOpening = false; - class _SqfliteDelegate extends DatabaseDelegate with _SqfliteExecutor { int _loadedSchemaVersion; @override @@ -53,43 +50,32 @@ class _SqfliteDelegate extends DatabaseDelegate with _SqfliteExecutor { @override Future open([GeneratedDatabase db]) async { - assert( - !_debugIsOpening && !isOpen, - 'Database opened multiple times, this should never happen. ' - 'Please report this at https://github.com/simolus3/moor/issues/135 to help fix this!' - 'Db already opened: $isOpen. Other operation in progress: $_debugIsOpening'); - _debugIsOpening = true; - - try { - String resolvedPath; - if (inDbFolder) { - resolvedPath = join(await s.getDatabasesPath(), path); - } else { - resolvedPath = path; - } - - final file = File(resolvedPath); - if (creator != null && !await file.exists()) { - await creator(file); - } - - // default value when no migration happened - _loadedSchemaVersion = db.schemaVersion; - - this.db = await s.openDatabase( - resolvedPath, - version: db.schemaVersion, - onCreate: (db, version) { - _loadedSchemaVersion = 0; - }, - onUpgrade: (db, from, to) { - _loadedSchemaVersion = from; - }, - singleInstance: singleInstance, - ); - } finally { - _debugIsOpening = false; + String resolvedPath; + if (inDbFolder) { + resolvedPath = join(await s.getDatabasesPath(), path); + } else { + resolvedPath = path; } + + final file = File(resolvedPath); + if (creator != null && !await file.exists()) { + await creator(file); + } + + // default value when no migration happened + _loadedSchemaVersion = db.schemaVersion; + + this.db = await s.openDatabase( + resolvedPath, + version: db.schemaVersion, + onCreate: (db, version) { + _loadedSchemaVersion = 0; + }, + onUpgrade: (db, from, to) { + _loadedSchemaVersion = from; + }, + singleInstance: singleInstance, + ); } @override From 6993b265326dc92079dbfe948f70373a1d1b7ed9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 4 Sep 2019 22:19:20 +0200 Subject: [PATCH 051/117] Release 1.7.2 of moor Fixed a bug where databases where not opened properly. --- moor/CHANGELOG.md | 4 ++++ moor/pubspec.yaml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 4bdcc249..3bee3386 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,3 +1,7 @@ +## 1.7.2 +- Fixed a race condition that caused the database to be opened multiple times on slower devices. + This problem was introduced in `1.7.0` and was causing problems during migrations. + ## 1.7.1 - Better documentation on `getSingle` and `watchSingle` for queries. - Fix `INTEGER NOT NULL PRIMARY KEY` wrongly requiring a value during insert (this never affected diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index aaf3dd7c..f2698b90 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -1,6 +1,6 @@ name: moor description: Moor is a safe and reactive persistence library for Dart applications -version: 1.7.1 +version: 1.7.2 repository: https://github.com/simolus3/moor homepage: https://moor.simonbinder.eu/ issue_tracker: https://github.com/simolus3/moor/issues @@ -18,7 +18,7 @@ dependencies: pedantic: ^1.0.0 dev_dependencies: - moor_generator: ^1.6.0 + moor_generator: ^1.7.0 build_runner: '>=1.3.0 <2.0.0' build_test: ^0.10.8 test: ^1.6.4 From 6ccaad866b7ee1684b629c27e26aa3c6270fb2ba Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 5 Sep 2019 10:52:33 +0200 Subject: [PATCH 052/117] Parse import statements in moor files --- sqlparser/lib/src/ast/ast.dart | 7 ++++++ .../lib/src/ast/moor/import_statement.dart | 20 +++++++++++++++ .../lib/src/ast/statements/statement.dart | 4 ++- sqlparser/lib/src/reader/parser/parser.dart | 25 +++++++++++++++++-- sqlparser/lib/src/reader/tokenizer/token.dart | 2 ++ .../test/parser/multiple_statements.dart | 18 +++++++++++++ 6 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 sqlparser/lib/src/ast/moor/import_statement.dart diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index b7b218f4..403ac59d 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -21,6 +21,8 @@ part 'expressions/subquery.dart'; part 'expressions/tuple.dart'; part 'expressions/variables.dart'; +part 'moor/import_statement.dart'; + part 'schema/column_definition.dart'; part 'schema/table_definition.dart'; @@ -174,6 +176,8 @@ abstract class AstVisitor { T visitNumberedVariable(NumberedVariable e); T visitNamedVariable(ColonNamedVariable e); + + T visitMoorImportStatement(ImportStatement e); } /// Visitor that walks down the entire tree, visiting all children in order. @@ -286,6 +290,9 @@ class RecursiveVisitor extends AstVisitor { @override T visitFrameSpec(FrameSpec e) => visitChildren(e); + @override + T visitMoorImportStatement(ImportStatement e) => visitChildren(e); + @protected T visitChildren(AstNode e) { for (var child in e.childNodes) { diff --git a/sqlparser/lib/src/ast/moor/import_statement.dart b/sqlparser/lib/src/ast/moor/import_statement.dart new file mode 100644 index 00000000..1fa6863f --- /dev/null +++ b/sqlparser/lib/src/ast/moor/import_statement.dart @@ -0,0 +1,20 @@ +part of '../ast.dart'; + +class ImportStatement extends Statement { + Token importToken; + StringLiteralToken importString; + final String importedFile; + + ImportStatement(this.importedFile); + + @override + T accept(AstVisitor visitor) {} + + @override + final Iterable childNodes = const []; + + @override + bool contentEquals(ImportStatement other) { + return other.importedFile == importedFile; + } +} diff --git a/sqlparser/lib/src/ast/statements/statement.dart b/sqlparser/lib/src/ast/statements/statement.dart index d4b2e88e..603352c0 100644 --- a/sqlparser/lib/src/ast/statements/statement.dart +++ b/sqlparser/lib/src/ast/statements/statement.dart @@ -1,6 +1,8 @@ part of '../ast.dart'; -abstract class Statement extends AstNode {} +abstract class Statement extends AstNode { + Token semicolon; +} /// Marker mixin for statements that read from an existing table structure. mixin CrudStatement on Statement {} diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index bc1053e1..340ee8bf 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -157,23 +157,44 @@ class Parser extends ParserBase Statement statement({bool expectEnd = true}) { final first = _peek; - final stmt = select() ?? + var stmt = select() ?? _deleteStmt() ?? _update() ?? _insertStmt() ?? _createTable(); + if (enableMoorExtensions) { + stmt ??= _import(); + } + if (stmt == null) { _error('Expected a sql statement to start here'); } - _matchOne(TokenType.semicolon); + if (_matchOne(TokenType.semicolon)) { + stmt.semicolon = _previous; + } + if (!_isAtEnd && expectEnd) { _error('Expected the statement to finish here'); } return stmt..setSpan(first, _previous); } + ImportStatement _import() { + if (_matchOne(TokenType.import)) { + final importToken = _previous; + final import = _consume(TokenType.stringLiteral, + 'Expected import file as a string literal (single quoted)') + as StringLiteralToken; + + return ImportStatement(import.value) + ..importToken = importToken + ..importString = import; + } + return null; + } + List statements() { final stmts = []; while (!_isAtEnd) { diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index 65f30733..c7ea7f00 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -140,6 +140,7 @@ enum TokenType { /// Moor specific token, used to declare a type converters mapped, inlineDart, + import, } const Map keywords = { @@ -238,6 +239,7 @@ const Map keywords = { const Map moorKeywords = { 'MAPPED': TokenType.mapped, + 'IMPORT': TokenType.import, }; class Token { diff --git a/sqlparser/test/parser/multiple_statements.dart b/sqlparser/test/parser/multiple_statements.dart index 507e0aa5..43b0e1f7 100644 --- a/sqlparser/test/parser/multiple_statements.dart +++ b/sqlparser/test/parser/multiple_statements.dart @@ -45,4 +45,22 @@ void main() { ), ); }); + + test('parses import directives in moor mode', () { + final sql = r''' + import 'test.dart'; + SELECT * FROM tbl; + '''; + + final tokens = Scanner(sql, scanMoorTokens: true).scanTokens(); + final statements = Parser(tokens, useMoor: true).statements(); + + expect(statements, hasLength(2)); + + final parsedImport = statements[0] as ImportStatement; + enforceEqual(parsedImport, ImportStatement('test.dart')); + expect(parsedImport.importToken, tokens[0]); + expect(parsedImport.importString, tokens[1]); + expect(parsedImport.semicolon, tokens[2]); + }); } From 71d1bfdc9c5041d9aaa8ce20dba2d83e0c7ef576 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 6 Sep 2019 22:23:44 +0200 Subject: [PATCH 053/117] Scaffold api to support type converters in moor files I still don't know how to parse expressions though :( --- .../analyzer/moor/inline_dart_resolver.dart | 45 +++++++++++++++++++ .../lib/src/analyzer/moor/parser.dart | 5 ++- moor_generator/lib/src/analyzer/session.dart | 6 ++- moor_generator/lib/src/backends/backend.dart | 2 + .../lib/src/backends/build/build_backend.dart | 7 +++ moor_generator/test/utils/test_backend.dart | 6 +++ 6 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart diff --git a/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart b/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart new file mode 100644 index 00000000..aee7b0a8 --- /dev/null +++ b/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart @@ -0,0 +1,45 @@ +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/element/type.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; + +/// Resolves the type of Dart expressions given as a string. The +/// [importStatements] are used to discover types. +/// +/// The way this works is that we create a fake file for the analyzer. That file +/// has the following content: +/// ``` +/// import 'package:moor/moor.dart'; // always imported +/// // all import statements +/// +/// var expr = $expression; +/// ``` +/// +/// We can then obtain the type of an expression by reading the inferred type +/// of the top-level `expr` variable in that source. +class InlineDartResolver { + final List importStatements = []; + final MoorTask task; + + InlineDartResolver(this.task); + + Future resolveDartTypeOf(String expression) async { + final template = _createDartTemplate(expression); + final unit = await task.backendTask.parseSource(template); + + final declaration = unit.declarations.single as TopLevelVariableDeclaration; + return declaration.variables.variables.single.initializer.staticType; + } + + String _createDartTemplate(String expression) { + final fakeDart = StringBuffer(); + + fakeDart.write("import 'package:moor/moor.dart';\n"); + for (var import in importStatements) { + fakeDart.write("import '$import';\n"); + } + + fakeDart.write('var expr = $expression;\n'); + + return fakeDart.toString(); + } +} diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index 50d2d8f9..05a7d927 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -16,7 +16,10 @@ class MoorParser { final createdReaders = []; for (var parsedStmt in results) { - if (parsedStmt.rootNode is CreateTableStatement) { + if (parsedStmt.rootNode is ImportStatement) { + final importStmt = (parsedStmt.rootNode) as ImportStatement; + task.inlineDartResolver.importStatements.add(importStmt.importedFile); + } else if (parsedStmt.rootNode is CreateTableStatement) { createdReaders.add(CreateTableReader(parsedStmt)); } else { task.reportError(ErrorInMoorFile( diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 79d2a42e..c8af9675 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -6,6 +6,7 @@ import 'package:analyzer/dart/element/type.dart'; import 'package:moor/moor.dart' show Table; import 'package:moor_generator/src/analyzer/dart/parser.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/moor/inline_dart_resolver.dart'; import 'package:moor_generator/src/analyzer/moor/parser.dart'; import 'package:moor_generator/src/analyzer/results.dart'; import 'package:moor_generator/src/analyzer/sql_queries/sql_parser.dart'; @@ -151,9 +152,12 @@ class DartTask extends FileTask { class MoorTask extends FileTask { final String content; final TypeMapper mapper = TypeMapper(); + /* late final */ InlineDartResolver inlineDartResolver; MoorTask(BackendTask task, MoorSession session, this.content) - : super(task, session); + : super(task, session) { + inlineDartResolver = InlineDartResolver(this); + } @override FutureOr compute() { diff --git a/moor_generator/lib/src/backends/backend.dart b/moor_generator/lib/src/backends/backend.dart index 4a5ec059..9b454f2d 100644 --- a/moor_generator/lib/src/backends/backend.dart +++ b/moor_generator/lib/src/backends/backend.dart @@ -1,3 +1,4 @@ +import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:logging/logging.dart'; import 'package:moor_generator/src/analyzer/session.dart'; @@ -17,5 +18,6 @@ abstract class BackendTask { Logger get log; Future resolveDart(Uri uri); + Future parseSource(String dart); Future readMoor(Uri uri); } diff --git a/moor_generator/lib/src/backends/build/build_backend.dart b/moor_generator/lib/src/backends/build/build_backend.dart index 825e1c3e..8867e961 100644 --- a/moor_generator/lib/src/backends/build/build_backend.dart +++ b/moor_generator/lib/src/backends/build/build_backend.dart @@ -1,3 +1,5 @@ +import 'package:analyzer/analyzer.dart'; +import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart' hide log; import 'package:build/build.dart' as build show log; @@ -32,6 +34,11 @@ class BuildBackendTask extends BackendTask { return step.resolver.libraryFor(_resolve(uri)); } + @override + Future parseSource(String dart) async { + return null; + } + @override Logger get log => build.log; } diff --git a/moor_generator/test/utils/test_backend.dart b/moor_generator/test/utils/test_backend.dart index 5a6ada24..3669e301 100644 --- a/moor_generator/test/utils/test_backend.dart +++ b/moor_generator/test/utils/test_backend.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart'; import 'package:build_test/build_test.dart'; @@ -60,4 +61,9 @@ class _TestBackendTask extends BackendTask { await backend._ready; return await backend._resolver.libraryFor(AssetId.resolve(path.toString())); } + + @override + Future parseSource(String dart) { + return null; + } } From 1130101f1f1813b6aa8a07ed412da903ba8b2c7f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 6 Sep 2019 22:37:15 +0200 Subject: [PATCH 054/117] Support writing string literals from Constant Fixes #137 --- .../src/runtime/expressions/variables.dart | 10 ++----- moor/lib/src/types/sql_types.dart | 10 +++++-- moor/test/expressions/constant_test.dart | 30 +++++++++++++++++++ 3 files changed, 39 insertions(+), 11 deletions(-) create mode 100644 moor/test/expressions/constant_test.dart diff --git a/moor/lib/src/runtime/expressions/variables.dart b/moor/lib/src/runtime/expressions/variables.dart index 53bb29f6..e91061c1 100644 --- a/moor/lib/src/runtime/expressions/variables.dart +++ b/moor/lib/src/runtime/expressions/variables.dart @@ -67,14 +67,8 @@ class Constant> extends Expression { @override void writeInto(GenerationContext context) { - // Instead of writing string literals (which we don't support because of - // possible sql injections), just write the variable. - if (value is String) { - _writeVariableIntoContext(context, value); - } else { - final type = context.typeSystem.forDartType(); - context.buffer.write(type.mapToSqlConstant(value)); - } + final type = context.typeSystem.forDartType(); + context.buffer.write(type.mapToSqlConstant(value)); } } diff --git a/moor/lib/src/types/sql_types.dart b/moor/lib/src/types/sql_types.dart index c912c372..cc59e113 100644 --- a/moor/lib/src/types/sql_types.dart +++ b/moor/lib/src/types/sql_types.dart @@ -56,9 +56,13 @@ class StringType extends SqlType { @override String mapToSqlConstant(String content) { - // TODO: implement mapToSqlConstant, we would probably have to take care - // of sql injection vulnerabilities here - throw UnimplementedError("Strings can't be mapped to sql literals yet"); + // From the sqlite docs: (https://www.sqlite.org/lang_expr.html) + // A string constant is formed by enclosing the string in single quotes ('). + // A single quote within the string can be encoded by putting two single + // quotes in a row - as in Pascal. C-style escapes using the backslash + // character are not supported because they are not standard SQL. + final escapedChars = content.replaceAll('\'', '\'\''); + return "'$escapedChars'"; } @override diff --git a/moor/test/expressions/constant_test.dart b/moor/test/expressions/constant_test.dart new file mode 100644 index 00000000..38b0818f --- /dev/null +++ b/moor/test/expressions/constant_test.dart @@ -0,0 +1,30 @@ +import 'package:moor/moor.dart'; +import 'package:moor/src/runtime/components/component.dart'; +import 'package:test_api/test_api.dart'; + +import '../data/tables/todos.dart'; + +void main() { + group('string literals', () { + test('can be written as constants', () { + testStringMapping('hello world', "'hello world'"); + }); + + test('supports escaping snigle quotes', () { + testStringMapping('what\'s that?', "'what\'\'s that?'"); + }); + + test('other chars are not escaped', () { + testStringMapping('\\\$"', "'\\\$\"'"); + }); + }); +} + +void testStringMapping(String dart, String expectedLiteral) { + final ctx = GenerationContext.fromDb(TodoDb(null)); + final constant = Constant(dart); + + constant.writeInto(ctx); + + expect(ctx.sql, expectedLiteral); +} From 10dca6a8a9ac63e2396df117d4914dfeb7fce414 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 15:19:40 +0200 Subject: [PATCH 055/117] Migrate analysis plugin to refactored generator --- moor_generator/lib/plugin.dart | 11 +- .../lib/src/analyzer/moor/parser.dart | 11 +- moor_generator/lib/src/analyzer/results.dart | 6 +- moor_generator/lib/src/analyzer/session.dart | 5 +- .../src/backends/plugin/backend/driver.dart | 128 ++++++++++++++++++ .../plugin/backend}/file_tracker.dart | 25 ++-- .../src/backends/plugin/backend/logger.dart | 24 ++++ .../plugin/backend/plugin_backend.dart | 36 +++++ .../lib/src/backends/plugin/plugin.dart | 64 +++++++++ .../plugin/services}/highlights/request.dart | 7 +- .../services}/highlights/sql_highlighter.dart | 8 +- .../src/plugin/analyzer/moor_analyzer.dart | 15 -- .../lib/src/plugin/analyzer/results.dart | 8 -- moor_generator/lib/src/plugin/driver.dart | 79 ----------- moor_generator/lib/src/plugin/plugin.dart | 66 --------- moor_generator/lib/src/plugin/starter.dart | 10 -- 16 files changed, 301 insertions(+), 202 deletions(-) create mode 100644 moor_generator/lib/src/backends/plugin/backend/driver.dart rename moor_generator/lib/src/{plugin/state => backends/plugin/backend}/file_tracker.dart (83%) create mode 100644 moor_generator/lib/src/backends/plugin/backend/logger.dart create mode 100644 moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart create mode 100644 moor_generator/lib/src/backends/plugin/plugin.dart rename moor_generator/lib/src/{plugin/analyzer => backends/plugin/services}/highlights/request.dart (64%) rename moor_generator/lib/src/{plugin/analyzer => backends/plugin/services}/highlights/sql_highlighter.dart (90%) delete mode 100644 moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart delete mode 100644 moor_generator/lib/src/plugin/analyzer/results.dart delete mode 100644 moor_generator/lib/src/plugin/driver.dart delete mode 100644 moor_generator/lib/src/plugin/plugin.dart delete mode 100644 moor_generator/lib/src/plugin/starter.dart diff --git a/moor_generator/lib/plugin.dart b/moor_generator/lib/plugin.dart index 6b21b8e9..bb72c4b5 100644 --- a/moor_generator/lib/plugin.dart +++ b/moor_generator/lib/plugin.dart @@ -1 +1,10 @@ -export 'src/plugin/starter.dart'; +import 'dart:isolate'; + +import 'package:analyzer/file_system/physical_file_system.dart'; +import 'package:analyzer_plugin/starter.dart'; +import 'package:moor_generator/src/backends/plugin/plugin.dart'; + +void start(List args, SendPort sendPort) { + ServerPluginStarter(MoorPlugin(PhysicalResourceProvider.INSTANCE)) + .start(sendPort); +} diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index 05a7d927..6be86cdb 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -10,8 +10,10 @@ class MoorParser { MoorParser(this.task); Future parseAndAnalyze() { + final engine = SqlEngine(useMoorExtensions: true); + final tokens = engine.tokenize(task.content); final results = - SqlEngine(useMoorExtensions: true).parseMultiple(task.content); + SqlEngine(useMoorExtensions: true).parseMultiple(tokens, task.content); final createdReaders = []; @@ -41,7 +43,12 @@ class MoorParser { final createdTables = createdReaders.map((r) => r.extractTable(task.mapper)).toList(); - final parsedFile = ParsedMoorFile(createdTables); + + final statements = + results.map((r) => r.rootNode).cast().toList(); + + final parsedFile = + ParsedMoorFile(tokens, statements, declaredTables: createdTables); return Future.value(parsedFile); } diff --git a/moor_generator/lib/src/analyzer/results.dart b/moor_generator/lib/src/analyzer/results.dart index 6b839f72..5e67e99a 100644 --- a/moor_generator/lib/src/analyzer/results.dart +++ b/moor_generator/lib/src/analyzer/results.dart @@ -3,6 +3,7 @@ import 'package:analyzer/dart/element/element.dart'; import 'package:moor_generator/src/model/specified_dao.dart'; import 'package:moor_generator/src/model/specified_database.dart'; import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:sqlparser/sqlparser.dart'; abstract class ParsedFile {} @@ -21,7 +22,10 @@ class ParsedDartFile extends ParsedFile { } class ParsedMoorFile extends ParsedFile { + final List tokens; + final List statements; final List declaredTables; - ParsedMoorFile(this.declaredTables); + ParsedMoorFile(this.tokens, this.statements, + {this.declaredTables = const []}); } diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index c8af9675..9aedfd14 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -154,6 +154,9 @@ class MoorTask extends FileTask { final TypeMapper mapper = TypeMapper(); /* late final */ InlineDartResolver inlineDartResolver; + ParsedMoorFile _lastResult; + ParsedMoorFile get lastResult => _lastResult; + MoorTask(BackendTask task, MoorSession session, this.content) : super(task, session) { inlineDartResolver = InlineDartResolver(this); @@ -162,6 +165,6 @@ class MoorTask extends FileTask { @override FutureOr compute() { final parser = MoorParser(this); - return parser.parseAndAnalyze(); + return parser.parseAndAnalyze().then((val) => _lastResult = val); } } diff --git a/moor_generator/lib/src/backends/plugin/backend/driver.dart b/moor_generator/lib/src/backends/plugin/backend/driver.dart new file mode 100644 index 00000000..75f72529 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/backend/driver.dart @@ -0,0 +1,128 @@ +// ignore_for_file: implementation_imports +import 'dart:async'; + +import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer/src/dart/analysis/file_state.dart'; +import 'package:analyzer/src/dart/analysis/driver.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; +import 'package:moor_generator/src/backends/plugin/backend/plugin_backend.dart'; + +class MoorDriver implements AnalysisDriverGeneric { + final FileTracker _tracker; + + final AnalysisDriverScheduler _scheduler; + final AnalysisDriver dartDriver; + + /// The content overlay exists so that we can perform up-to-date analysis on + /// unsaved files. + final FileContentOverlay contentOverlay; + final ResourceProvider _resourceProvider; + + final MoorSession session = MoorSession(); + + MoorDriver(this._tracker, this._scheduler, this.dartDriver, + this.contentOverlay, this._resourceProvider) { + _scheduler.add(this); + } + + bool _ownsFile(String path) => path.endsWith('.moor'); + + @override + void addFile(String path) { + if (_ownsFile(path)) { + _tracker.addFile(path); + } + } + + @override + void dispose() { + _scheduler.remove(this); + dartDriver.dispose(); + } + + void handleFileChanged(String path) { + if (_ownsFile(path)) { + _tracker.handleContentChanged(path); + _scheduler.notify(this); + } + } + + @override + bool get hasFilesToAnalyze => _tracker.hasWork; + + @override + Future performWork() async { + final completer = Completer(); + + if (_tracker.hasWork) { + _tracker.work((path) async { + try { + final backendTask = _createTask(path); + final moorTask = await session.startMoorTask(backendTask); + await moorTask.compute(); + + return moorTask; + } finally { + completer.complete(); + } + }); + + await completer.future; + } + } + + String readFile(String path) { + final overlay = contentOverlay[path]; + if (overlay != null) { + return overlay; + } + + final file = _resourceProvider.getFile(path); + return file.exists ? file.readAsStringSync() : ''; + } + + /// Finds the absolute path of a [reference] url, optionally assuming that the + /// [reference] appears in [base]. This supports both "package:"-based uris + /// and relative imports. + String absolutePath(Uri reference, {Uri base}) { + final factory = dartDriver.sourceFactory; + final baseSource = base == null ? null : factory.forUri2(base); + final source = + dartDriver.sourceFactory.resolveUri(baseSource, reference.toString()); + return source.fullName; + } + + PluginTask _createTask(String path) { + final uri = Uri.parse(path); + return PluginTask(uri, this); + } + + @override + set priorityFiles(List priorityPaths) { + _tracker.setPriorityFiles(priorityPaths); + } + + @override + AnalysisDriverPriority get workPriority { + if (_tracker.hasWork) { + final mostImportant = _tracker.fileWithHighestPriority; + switch (mostImportant.currentPriority) { + case FilePriority.ignore: + return AnalysisDriverPriority.nothing; + case FilePriority.regular: + return AnalysisDriverPriority.general; + case FilePriority.interactive: + return AnalysisDriverPriority.interactive; + } + } else { + return AnalysisDriverPriority.nothing; + } + throw AssertionError('unreachable'); + } + + Future parseMoorFile(String path) { + _scheduler.notify(this); + return _tracker.results(path); + } +} diff --git a/moor_generator/lib/src/plugin/state/file_tracker.dart b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart similarity index 83% rename from moor_generator/lib/src/plugin/state/file_tracker.dart rename to moor_generator/lib/src/backends/plugin/backend/file_tracker.dart index 42c982b2..80289878 100644 --- a/moor_generator/lib/src/plugin/state/file_tracker.dart +++ b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart @@ -1,7 +1,13 @@ import 'dart:async'; import 'package:collection/collection.dart'; -import 'package:moor_generator/src/plugin/analyzer/results.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; + +int _compareByPriority(TrackedFile a, TrackedFile b) { + final aPriority = a.currentPriority?.index ?? 0; + final bPriority = b.currentPriority?.index ?? 0; + return aPriority.compareTo(bPriority); +} /// Keeps track of files that need to be analyzed by the moor plugin. class FileTracker { @@ -13,12 +19,6 @@ class FileTracker { _pendingWork = PriorityQueue(_compareByPriority); } - int _compareByPriority(TrackedFile a, TrackedFile b) { - final aPriority = a.currentPriority?.index ?? 0; - final bPriority = b.currentPriority?.index ?? 0; - return aPriority.compareTo(bPriority); - } - void _updateFile(TrackedFile file, Function(TrackedFile) update) { _pendingWork.remove(file); update(file); @@ -32,6 +32,7 @@ class FileTracker { } bool get hasWork => _pendingWork.isNotEmpty; + TrackedFile get fileWithHighestPriority => _pendingWork.first; TrackedFile addFile(String path) { return _trackedFiles.putIfAbsent(path, () { @@ -64,19 +65,19 @@ class FileTracker { _putInQueue(tracked); } - Future results(String path) async { + Future results(String path) async { final tracked = addFile(path); if (tracked._currentResult != null) { return tracked._currentResult; } else { - final completer = Completer(); + final completer = Completer(); tracked._waiting.add(completer); return completer.future; } } - void work(Future Function(String path) worker) { + void work(Future Function(String path) worker) { if (_pendingWork.isNotEmpty) { final unit = _pendingWork.removeFirst(); @@ -111,8 +112,8 @@ class TrackedFile { /// Whether this file has been given an elevated priority, for instance /// because the user is currently typing in it. bool _prioritized = false; - MoorAnalysisResults _currentResult; - final List> _waiting = []; + MoorTask _currentResult; + final List> _waiting = []; FilePriority get currentPriority => _prioritized ? FilePriority.interactive : defaultPriority; diff --git a/moor_generator/lib/src/backends/plugin/backend/logger.dart b/moor_generator/lib/src/backends/plugin/backend/logger.dart new file mode 100644 index 00000000..b9da0212 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/backend/logger.dart @@ -0,0 +1,24 @@ +import 'package:analyzer_plugin/protocol/protocol_generated.dart'; +import 'package:logging/logging.dart'; +import 'package:moor_generator/src/backends/plugin/plugin.dart'; + +var _initialized = false; + +/// Configures the [Logger.root] logger to work with the plugin. Sadly, we don't +/// really have a way to view [print] outputs from plugins, so we use the +/// diagnostics notification for that. +void setupLogger(MoorPlugin plugin) { + assert(!_initialized, 'Logger initialized multiple times'); + + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((LogRecord rec) { + if (rec.level >= Level.INFO) { + final isFatal = rec.level >= Level.WARNING; + final error = + PluginErrorParams(isFatal, rec.message, rec.stackTrace?.toString()); + + plugin.channel.sendNotification(error.toNotification()); + } + }); + _initialized = true; +} diff --git a/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart b/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart new file mode 100644 index 00000000..bf55105d --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart @@ -0,0 +1,36 @@ +import 'package:analyzer/dart/ast/ast.dart'; +import 'package:analyzer/dart/element/element.dart'; +import 'package:logging/logging.dart'; +import 'package:moor_generator/src/backends/backend.dart'; + +import 'driver.dart'; + +class PluginBackend extends Backend {} + +class PluginTask extends BackendTask { + @override + final Uri entrypoint; + final MoorDriver driver; + + PluginTask(this.entrypoint, this.driver); + + @override + final Logger log = Logger.root; + + @override + Future parseSource(String dart) { + return null; + } + + @override + Future readMoor(Uri uri) async { + final path = driver.absolutePath(uri, base: entrypoint); + return driver.readFile(path); + } + + @override + Future resolveDart(Uri uri) { + final path = driver.absolutePath(uri, base: entrypoint); + return driver.dartDriver.currentSession.getLibraryByUri(path); + } +} diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart new file mode 100644 index 00000000..71ad07a1 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -0,0 +1,64 @@ +import 'package:analyzer/context/context_root.dart'; +// ignore: implementation_imports +import 'package:analyzer/src/context/builder.dart'; +import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/plugin/plugin.dart'; +import 'package:analyzer_plugin/protocol/protocol_generated.dart' as plugin; +import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; + +import 'backend/driver.dart'; +import 'backend/logger.dart'; + +class MoorPlugin extends ServerPlugin { + MoorPlugin(ResourceProvider provider) : super(provider) { + setupLogger(this); + } + + @override + final List fileGlobsToAnalyze = const ['*.moor']; + @override + final String name = 'Moor plugin'; + @override + // docs say that this should a version of _this_ plugin, but they lie. this + // version will be used to determine compatibility with the analyzer + final String version = '2.0.0-alpha.0'; + @override + final String contactInfo = + 'Create an issue at https://github.com/simolus3/moor/'; + + @override + MoorDriver createAnalysisDriver(plugin.ContextRoot contextRoot) { + // create an analysis driver we can use to resolve Dart files + final analyzerRoot = ContextRoot(contextRoot.root, contextRoot.exclude, + pathContext: resourceProvider.pathContext) + ..optionsFilePath = contextRoot.optionsFile; + + final builder = ContextBuilder(resourceProvider, sdkManager, null) + ..analysisDriverScheduler = analysisDriverScheduler + ..byteStore = byteStore + ..performanceLog = performanceLog + ..fileContentOverlay = fileContentOverlay; + + // todo we listen because we copied this from the angular plugin. figure out + // why exactly this is necessary + final dartDriver = builder.buildDriver(analyzerRoot) + ..results.listen((_) {}) // Consume the stream, otherwise we leak. + ..exceptions.listen((_) {}); // Consume the stream, otherwise we leak.; + + final tracker = FileTracker(); + return MoorDriver(tracker, analysisDriverScheduler, dartDriver, + fileContentOverlay, resourceProvider); + } + + @override + void contentChanged(String path) { + _moorDriverForPath(path)?.handleFileChanged(path); + } + + MoorDriver _moorDriverForPath(String path) { + final driver = super.driverForPath(path); + + if (driver is! MoorDriver) return null; + return driver as MoorDriver; + } +} diff --git a/moor_generator/lib/src/plugin/analyzer/highlights/request.dart b/moor_generator/lib/src/backends/plugin/services/highlights/request.dart similarity index 64% rename from moor_generator/lib/src/plugin/analyzer/highlights/request.dart rename to moor_generator/lib/src/backends/plugin/services/highlights/request.dart index 65f9fbee..ad730ed6 100644 --- a/moor_generator/lib/src/plugin/analyzer/highlights/request.dart +++ b/moor_generator/lib/src/backends/plugin/services/highlights/request.dart @@ -1,14 +1,13 @@ import 'package:analyzer/file_system/file_system.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; - -import '../results.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; class MoorHighlightingRequest extends HighlightsRequest { @override final String path; @override final ResourceProvider resourceProvider; - final MoorAnalysisResults parsedFile; + final MoorTask task; - MoorHighlightingRequest(this.parsedFile, this.path, this.resourceProvider); + MoorHighlightingRequest(this.task, this.path, this.resourceProvider); } diff --git a/moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart b/moor_generator/lib/src/backends/plugin/services/highlights/sql_highlighter.dart similarity index 90% rename from moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart rename to moor_generator/lib/src/backends/plugin/services/highlights/sql_highlighter.dart index 4eb3f0b1..140e3754 100644 --- a/moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart +++ b/moor_generator/lib/src/backends/plugin/services/highlights/sql_highlighter.dart @@ -1,6 +1,6 @@ import 'package:analyzer_plugin/protocol/protocol_common.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; -import 'package:moor_generator/src/plugin/analyzer/highlights/request.dart'; +import 'package:moor_generator/src/backends/plugin/services/highlights/request.dart'; import 'package:sqlparser/sqlparser.dart'; const _notBuiltIn = { @@ -37,11 +37,13 @@ class SqlHighlighter implements HighlightsContributor { final typedRequest = request as MoorHighlightingRequest; final visitor = _HighlightingVisitor(collector); - for (var stmt in typedRequest.parsedFile.statements) { + final result = typedRequest.task.lastResult; + + for (var stmt in result.statements) { stmt.accept(visitor); } - for (var token in typedRequest.parsedFile.sqlTokens) { + for (var token in result.tokens) { if (!_notBuiltIn.contains(token.type)) { final start = token.span.start.offset; final length = token.span.length; diff --git a/moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart b/moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart deleted file mode 100644 index a5456bfd..00000000 --- a/moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart +++ /dev/null @@ -1,15 +0,0 @@ -import 'package:analyzer/file_system/file_system.dart'; -import 'package:moor_generator/src/plugin/analyzer/results.dart'; -import 'package:sqlparser/sqlparser.dart'; - -class MoorAnalyzer { - Future analyze(File file) async { - final content = file.readAsStringSync(); - final sqlEngine = SqlEngine(); - - final tokens = sqlEngine.tokenize(content); - final stmts = sqlEngine.parseMultiple(tokens, content); - - return MoorAnalysisResults(stmts.map((r) => r.rootNode).toList(), tokens); - } -} diff --git a/moor_generator/lib/src/plugin/analyzer/results.dart b/moor_generator/lib/src/plugin/analyzer/results.dart deleted file mode 100644 index 9aa4cc2b..00000000 --- a/moor_generator/lib/src/plugin/analyzer/results.dart +++ /dev/null @@ -1,8 +0,0 @@ -import 'package:sqlparser/sqlparser.dart'; - -class MoorAnalysisResults { - final List statements; - final List sqlTokens; - - MoorAnalysisResults(this.statements, this.sqlTokens); -} diff --git a/moor_generator/lib/src/plugin/driver.dart b/moor_generator/lib/src/plugin/driver.dart deleted file mode 100644 index 3f16e40d..00000000 --- a/moor_generator/lib/src/plugin/driver.dart +++ /dev/null @@ -1,79 +0,0 @@ -// ignore_for_file: implementation_imports -import 'dart:async'; - -import 'package:analyzer/file_system/file_system.dart'; -import 'package:analyzer/src/dart/analysis/driver.dart'; -import 'package:moor_generator/src/plugin/state/file_tracker.dart'; - -import 'analyzer/moor_analyzer.dart'; -import 'analyzer/results.dart'; - -class MoorDriver implements AnalysisDriverGeneric { - final FileTracker _tracker; - final AnalysisDriverScheduler _scheduler; - final MoorAnalyzer _analyzer; - final ResourceProvider _resources; - - MoorDriver(this._tracker, this._scheduler, this._analyzer, this._resources) { - _scheduler.add(this); - } - - bool _ownsFile(String path) => path.endsWith('.moor'); - - @override - void addFile(String path) { - if (_ownsFile(path)) { - _tracker.addFile(path); - } - } - - @override - void dispose() { - _scheduler.remove(this); - } - - void handleFileChanged(String path) { - if (_ownsFile(path)) { - _tracker.handleContentChanged(path); - _scheduler.notify(this); - } - } - - @override - bool get hasFilesToAnalyze => _tracker.hasWork; - - @override - Future performWork() async { - final completer = Completer(); - - if (_tracker.hasWork) { - _tracker.work((path) { - try { - return _resolveMoorFile(path); - } finally { - completer.complete(); - } - }); - - await completer.future; - } - } - - Future _resolveMoorFile(String path) { - return _analyzer.analyze(_resources.getFile(path)); - } - - @override - set priorityFiles(List priorityPaths) { - _tracker.setPriorityFiles(priorityPaths); - } - - @override - // todo ask the tracker about the top-priority file. - AnalysisDriverPriority get workPriority => AnalysisDriverPriority.general; - - Future parseMoorFile(String path) { - _scheduler.notify(this); - return _tracker.results(path); - } -} diff --git a/moor_generator/lib/src/plugin/plugin.dart b/moor_generator/lib/src/plugin/plugin.dart deleted file mode 100644 index 3d6eacda..00000000 --- a/moor_generator/lib/src/plugin/plugin.dart +++ /dev/null @@ -1,66 +0,0 @@ -import 'package:analyzer/file_system/file_system.dart'; -import 'package:analyzer_plugin/plugin/highlights_mixin.dart'; -import 'package:analyzer_plugin/plugin/plugin.dart'; -import 'package:analyzer_plugin/protocol/protocol.dart'; -import 'package:analyzer_plugin/protocol/protocol_generated.dart'; -import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; -import 'package:moor_generator/src/plugin/state/file_tracker.dart'; - -import 'analyzer/highlights/request.dart'; -import 'analyzer/highlights/sql_highlighter.dart'; -import 'analyzer/moor_analyzer.dart'; -import 'driver.dart'; - -class MoorPlugin extends ServerPlugin with HighlightsMixin { - MoorPlugin(ResourceProvider provider) : super(provider); - - @override - final List fileGlobsToAnalyze = const ['*.moor']; - @override - final String name = 'Moor plugin'; - @override - // docs say that this should a version of _this_ plugin, but they lie. this - // version will be used to determine compatibility with the analyzer - final String version = '2.0.0-alpha.0'; - @override - final String contactInfo = - 'Create an issue at https://github.com/simolus3/moor/'; - - @override - MoorDriver createAnalysisDriver(ContextRoot contextRoot) { - final tracker = FileTracker(); - final analyzer = MoorAnalyzer(); - return MoorDriver( - tracker, analysisDriverScheduler, analyzer, resourceProvider); - } - - @override - void contentChanged(String path) { - _moorDriverForPath(path)?.handleFileChanged(path); - } - - MoorDriver _moorDriverForPath(String path) { - final driver = super.driverForPath(path); - - if (driver is! MoorDriver) return null; - return driver as MoorDriver; - } - - @override - List getHighlightsContributors(String path) { - return const [SqlHighlighter()]; - } - - @override - Future getHighlightsRequest(String path) async { - final driver = _moorDriverForPath(path); - if (driver == null) { - throw RequestFailure( - RequestErrorFactory.pluginError('Not driver set for path', null)); - } - - final parsed = await driver.parseMoorFile(path); - - return MoorHighlightingRequest(parsed, path, resourceProvider); - } -} diff --git a/moor_generator/lib/src/plugin/starter.dart b/moor_generator/lib/src/plugin/starter.dart deleted file mode 100644 index 815b31ce..00000000 --- a/moor_generator/lib/src/plugin/starter.dart +++ /dev/null @@ -1,10 +0,0 @@ -import 'dart:isolate'; - -import 'package:analyzer/file_system/physical_file_system.dart'; -import 'package:analyzer_plugin/starter.dart'; -import 'package:moor_generator/src/plugin/plugin.dart'; - -void start(List args, SendPort sendPort) { - ServerPluginStarter(MoorPlugin(PhysicalResourceProvider.INSTANCE)) - .start(sendPort); -} From 1fcc6facee6ad47da8f7e6f5b3fff434507ef041 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 18:49:11 +0200 Subject: [PATCH 056/117] Implement basic functionality of the analyzer plugin --- .../lib/src/backends/build/build_backend.dart | 1 - .../src/backends/plugin/backend/driver.dart | 5 +- .../backends/plugin/backend/file_tracker.dart | 2 +- .../src/backends/plugin/backend/logger.dart | 4 +- .../lib/src/backends/plugin/plugin.dart | 52 +++++++++++++++-- .../src/backends/plugin/services/folding.dart | 38 +++++++++++++ .../sql_highlighter.dart => highlights.dart} | 35 +++--------- .../plugin/services/highlights/request.dart | 13 ----- .../src/backends/plugin/services/outline.dart | 57 +++++++++++++++++++ .../backends/plugin/services/requests.dart | 16 ++++++ .../lib/src/ast/moor/import_statement.dart | 2 +- .../lib/src/ast/statements/create_table.dart | 3 + sqlparser/lib/src/reader/parser/schema.dart | 10 +++- 13 files changed, 183 insertions(+), 55 deletions(-) create mode 100644 moor_generator/lib/src/backends/plugin/services/folding.dart rename moor_generator/lib/src/backends/plugin/services/{highlights/sql_highlighter.dart => highlights.dart} (65%) delete mode 100644 moor_generator/lib/src/backends/plugin/services/highlights/request.dart create mode 100644 moor_generator/lib/src/backends/plugin/services/outline.dart create mode 100644 moor_generator/lib/src/backends/plugin/services/requests.dart diff --git a/moor_generator/lib/src/backends/build/build_backend.dart b/moor_generator/lib/src/backends/build/build_backend.dart index 8867e961..6fa105ea 100644 --- a/moor_generator/lib/src/backends/build/build_backend.dart +++ b/moor_generator/lib/src/backends/build/build_backend.dart @@ -1,4 +1,3 @@ -import 'package:analyzer/analyzer.dart'; import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart' hide log; diff --git a/moor_generator/lib/src/backends/plugin/backend/driver.dart b/moor_generator/lib/src/backends/plugin/backend/driver.dart index 75f72529..b141cf4a 100644 --- a/moor_generator/lib/src/backends/plugin/backend/driver.dart +++ b/moor_generator/lib/src/backends/plugin/backend/driver.dart @@ -88,19 +88,20 @@ class MoorDriver implements AnalysisDriverGeneric { String absolutePath(Uri reference, {Uri base}) { final factory = dartDriver.sourceFactory; final baseSource = base == null ? null : factory.forUri2(base); + final source = dartDriver.sourceFactory.resolveUri(baseSource, reference.toString()); return source.fullName; } PluginTask _createTask(String path) { - final uri = Uri.parse(path); + final uri = Uri.parse(path).replace(scheme: 'file'); return PluginTask(uri, this); } @override set priorityFiles(List priorityPaths) { - _tracker.setPriorityFiles(priorityPaths); + _tracker.setPriorityFiles(priorityPaths.where(_ownsFile)); } @override diff --git a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart index 80289878..6e31ef5e 100644 --- a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart +++ b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart @@ -46,7 +46,7 @@ class FileTracker { _putInQueue(addFile(path)); } - void setPriorityFiles(List priority) { + void setPriorityFiles(Iterable priority) { // remove prioritized flag from existing files for (var file in _currentPriority) { _updateFile(file, (f) => f._prioritized = false); diff --git a/moor_generator/lib/src/backends/plugin/backend/logger.dart b/moor_generator/lib/src/backends/plugin/backend/logger.dart index b9da0212..c7008673 100644 --- a/moor_generator/lib/src/backends/plugin/backend/logger.dart +++ b/moor_generator/lib/src/backends/plugin/backend/logger.dart @@ -13,9 +13,9 @@ void setupLogger(MoorPlugin plugin) { Logger.root.level = Level.ALL; Logger.root.onRecord.listen((LogRecord rec) { if (rec.level >= Level.INFO) { - final isFatal = rec.level >= Level.WARNING; + final isFatal = rec.level > Level.WARNING; final error = - PluginErrorParams(isFatal, rec.message, rec.stackTrace?.toString()); + PluginErrorParams(isFatal, rec.message, rec.stackTrace.toString()); plugin.channel.sendNotification(error.toNotification()); } diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart index 71ad07a1..7145d3fb 100644 --- a/moor_generator/lib/src/backends/plugin/plugin.dart +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -1,15 +1,25 @@ -import 'package:analyzer/context/context_root.dart'; -// ignore: implementation_imports -import 'package:analyzer/src/context/builder.dart'; +import 'package:analyzer/src/context/context_root.dart'; // ignore: implementation_imports +import 'package:analyzer/src/context/builder.dart'; // ignore: implementation_imports import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/plugin/folding_mixin.dart'; +import 'package:analyzer_plugin/plugin/highlights_mixin.dart'; +import 'package:analyzer_plugin/plugin/outline_mixin.dart'; import 'package:analyzer_plugin/plugin/plugin.dart'; import 'package:analyzer_plugin/protocol/protocol_generated.dart' as plugin; +import 'package:analyzer_plugin/utilities/folding/folding.dart'; +import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; +import 'package:analyzer_plugin/utilities/outline/outline.dart'; import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; +import 'package:moor_generator/src/backends/plugin/services/folding.dart'; +import 'package:moor_generator/src/backends/plugin/services/highlights.dart'; +import 'package:moor_generator/src/backends/plugin/services/outline.dart'; +import 'package:moor_generator/src/backends/plugin/services/requests.dart'; import 'backend/driver.dart'; import 'backend/logger.dart'; -class MoorPlugin extends ServerPlugin { +class MoorPlugin extends ServerPlugin + with OutlineMixin, HighlightsMixin, FoldingMixin { MoorPlugin(ResourceProvider provider) : super(provider) { setupLogger(this); } @@ -61,4 +71,38 @@ class MoorPlugin extends ServerPlugin { if (driver is! MoorDriver) return null; return driver as MoorDriver; } + + Future _createMoorRequest(String path) async { + final driver = _moorDriverForPath(path); + final task = await driver.parseMoorFile(path); + + return MoorRequest(task, resourceProvider); + } + + @override + List getOutlineContributors(String path) { + return const [MoorOutlineContributor()]; + } + + @override + Future getOutlineRequest(String path) => + _createMoorRequest(path); + + @override + List getHighlightsContributors(String path) { + return const [MoorHighlightContributor()]; + } + + @override + Future getHighlightsRequest(String path) => + _createMoorRequest(path); + + @override + List getFoldingContributors(String path) { + return const [MoorFoldingContributor()]; + } + + @override + Future getFoldingRequest(String path) => + _createMoorRequest(path); } diff --git a/moor_generator/lib/src/backends/plugin/services/folding.dart b/moor_generator/lib/src/backends/plugin/services/folding.dart new file mode 100644 index 00000000..36f65b1d --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/folding.dart @@ -0,0 +1,38 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:analyzer_plugin/utilities/folding/folding.dart'; +import 'package:moor_generator/src/backends/plugin/services/requests.dart'; +import 'package:sqlparser/sqlparser.dart'; + +class MoorFoldingContributor implements FoldingContributor { + const MoorFoldingContributor(); + + @override + void computeFolding(FoldingRequest request, FoldingCollector collector) { + final moorRequest = request as MoorRequest; + + final visitor = _FoldingVisitor(collector); + for (var stmt in moorRequest.resolvedTask.lastResult.statements) { + stmt.accept(visitor); + } + } +} + +class _FoldingVisitor extends RecursiveVisitor { + final FoldingCollector collector; + + _FoldingVisitor(this.collector); + + @override + void visitCreateTableStatement(CreateTableStatement e) { + final startBody = e.openingBracket; + final endBody = e.closingBracket; + + // report everything between the two brackets as class body + final first = startBody.span.end.offset + 1; + final last = endBody.span.start.offset - 1; + + if (last - first < 0) return; // empty body, e.g. CREATE TABLE () + + collector.addRegion(first, last - first, FoldingKind.CLASS_BODY); + } +} diff --git a/moor_generator/lib/src/backends/plugin/services/highlights/sql_highlighter.dart b/moor_generator/lib/src/backends/plugin/services/highlights.dart similarity index 65% rename from moor_generator/lib/src/backends/plugin/services/highlights/sql_highlighter.dart rename to moor_generator/lib/src/backends/plugin/services/highlights.dart index 140e3754..6f10528e 100644 --- a/moor_generator/lib/src/backends/plugin/services/highlights/sql_highlighter.dart +++ b/moor_generator/lib/src/backends/plugin/services/highlights.dart @@ -1,50 +1,29 @@ import 'package:analyzer_plugin/protocol/protocol_common.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; -import 'package:moor_generator/src/backends/plugin/services/highlights/request.dart'; +import 'package:moor_generator/src/backends/plugin/services/requests.dart'; import 'package:sqlparser/sqlparser.dart'; -const _notBuiltIn = { - TokenType.numberLiteral, - TokenType.stringLiteral, - TokenType.identifier, - TokenType.leftParen, - TokenType.rightParen, - TokenType.comma, - TokenType.star, - TokenType.less, - TokenType.lessEqual, - TokenType.lessMore, - TokenType.equal, - TokenType.more, - TokenType.moreEqual, - TokenType.shiftRight, - TokenType.shiftLeft, - TokenType.exclamationEqual, - TokenType.plus, - TokenType.minus, -}; - -class SqlHighlighter implements HighlightsContributor { - const SqlHighlighter(); +class MoorHighlightContributor implements HighlightsContributor { + const MoorHighlightContributor(); @override void computeHighlights( HighlightsRequest request, HighlightsCollector collector) { - if (request is! MoorHighlightingRequest) { + if (request is! MoorRequest) { return; } - final typedRequest = request as MoorHighlightingRequest; + final typedRequest = request as MoorRequest; final visitor = _HighlightingVisitor(collector); - final result = typedRequest.task.lastResult; + final result = typedRequest.resolvedTask.lastResult; for (var stmt in result.statements) { stmt.accept(visitor); } for (var token in result.tokens) { - if (!_notBuiltIn.contains(token.type)) { + if (token is KeywordToken) { final start = token.span.start.offset; final length = token.span.length; collector.addRegion(start, length, HighlightRegionType.BUILT_IN); diff --git a/moor_generator/lib/src/backends/plugin/services/highlights/request.dart b/moor_generator/lib/src/backends/plugin/services/highlights/request.dart deleted file mode 100644 index ad730ed6..00000000 --- a/moor_generator/lib/src/backends/plugin/services/highlights/request.dart +++ /dev/null @@ -1,13 +0,0 @@ -import 'package:analyzer/file_system/file_system.dart'; -import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; - -class MoorHighlightingRequest extends HighlightsRequest { - @override - final String path; - @override - final ResourceProvider resourceProvider; - final MoorTask task; - - MoorHighlightingRequest(this.task, this.path, this.resourceProvider); -} diff --git a/moor_generator/lib/src/backends/plugin/services/outline.dart b/moor_generator/lib/src/backends/plugin/services/outline.dart new file mode 100644 index 00000000..5335cab9 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/outline.dart @@ -0,0 +1,57 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:analyzer_plugin/utilities/outline/outline.dart'; +import 'package:moor_generator/src/backends/plugin/services/requests.dart'; +import 'package:sqlparser/sqlparser.dart'; + +const _defaultFlags = 0; + +class MoorOutlineContributor implements OutlineContributor { + const MoorOutlineContributor(); + + @override + void computeOutline(OutlineRequest request, OutlineCollector collector) { + final moorRequest = request as MoorRequest; + final file = moorRequest.path; + + final libraryElement = Element(ElementKind.FILE, file, _defaultFlags); + collector.startElement( + libraryElement, 0, moorRequest.resolvedTask.content.length); + + final visitor = _OutlineVisitor(collector); + for (var stmt in moorRequest.resolvedTask.lastResult.statements) { + stmt.accept(visitor); + } + + collector.endElement(); + } +} + +class _OutlineVisitor extends RecursiveVisitor { + final OutlineCollector collector; + + _OutlineVisitor(this.collector); + + Element _startElement(ElementKind kind, String name, AstNode e) { + final element = Element(kind, name, _defaultFlags); + + final offset = e.firstPosition; + final length = e.lastPosition - offset; + collector.startElement(element, offset, length); + + return element; + } + + @override + void visitCreateTableStatement(CreateTableStatement e) { + _startElement(ElementKind.CLASS, e.tableName, e); + super.visitChildren(e); + collector.endElement(); + } + + @override + void visitColumnDefinition(ColumnDefinition e) { + _startElement(ElementKind.FIELD, e.columnName, e); + super.visitChildren(e); + collector.endElement(); + } +} diff --git a/moor_generator/lib/src/backends/plugin/services/requests.dart b/moor_generator/lib/src/backends/plugin/services/requests.dart new file mode 100644 index 00000000..bfd22ee4 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/requests.dart @@ -0,0 +1,16 @@ +import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/utilities/folding/folding.dart'; +import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; +import 'package:analyzer_plugin/utilities/outline/outline.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; + +class MoorRequest implements OutlineRequest, HighlightsRequest, FoldingRequest { + final MoorTask resolvedTask; + @override + final ResourceProvider resourceProvider; + + MoorRequest(this.resolvedTask, this.resourceProvider); + + @override + String get path => resolvedTask.backendTask.entrypoint.toFilePath(); +} diff --git a/sqlparser/lib/src/ast/moor/import_statement.dart b/sqlparser/lib/src/ast/moor/import_statement.dart index 1fa6863f..31115896 100644 --- a/sqlparser/lib/src/ast/moor/import_statement.dart +++ b/sqlparser/lib/src/ast/moor/import_statement.dart @@ -8,7 +8,7 @@ class ImportStatement extends Statement { ImportStatement(this.importedFile); @override - T accept(AstVisitor visitor) {} + T accept(AstVisitor visitor) => visitor.visitMoorImportStatement(this); @override final Iterable childNodes = const []; diff --git a/sqlparser/lib/src/ast/statements/create_table.dart b/sqlparser/lib/src/ast/statements/create_table.dart index ee000b28..961a9659 100644 --- a/sqlparser/lib/src/ast/statements/create_table.dart +++ b/sqlparser/lib/src/ast/statements/create_table.dart @@ -9,6 +9,9 @@ class CreateTableStatement extends Statement with SchemaStatement { final List tableConstraints; final bool withoutRowId; + Token openingBracket; + Token closingBracket; + CreateTableStatement( {this.ifNotExists = false, @required this.tableName, diff --git a/sqlparser/lib/src/reader/parser/schema.dart b/sqlparser/lib/src/reader/parser/schema.dart index fc4c7551..e33e7335 100644 --- a/sqlparser/lib/src/reader/parser/schema.dart +++ b/sqlparser/lib/src/reader/parser/schema.dart @@ -19,7 +19,7 @@ mixin SchemaParser on ParserBase { _consumeIdentifier('Expected a table name', lenient: true); // we don't currently support CREATE TABLE x AS SELECT ... statements - _consume( + final leftParen = _consume( TokenType.leftParen, 'Expected opening parenthesis to list columns'); final columns = []; @@ -42,7 +42,8 @@ mixin SchemaParser on ParserBase { } } while (_matchOne(TokenType.comma)); - _consume(TokenType.rightParen, 'Expected closing parenthesis'); + final rightParen = + _consume(TokenType.rightParen, 'Expected closing parenthesis'); var withoutRowId = false; if (_matchOne(TokenType.without)) { @@ -57,7 +58,10 @@ mixin SchemaParser on ParserBase { withoutRowId: withoutRowId, columns: columns, tableConstraints: tableConstraints, - )..setSpan(first, _previous); + ) + ..setSpan(first, _previous) + ..openingBracket = leftParen + ..closingBracket = rightParen; } ColumnDefinition _columnDefinition() { From 82688673692e5b55329e1d5d25a3c1d230c005a3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 20:01:33 +0200 Subject: [PATCH 057/117] Fix analyzer errors, re-run build_runner --- .../tests/lib/database/database.g.dart | 318 ++++++++++-------- .../lib/src/backends/build/build_backend.dart | 1 - .../lib/src/ast/moor/import_statement.dart | 2 +- 3 files changed, 176 insertions(+), 145 deletions(-) diff --git a/extras/integration_tests/tests/lib/database/database.g.dart b/extras/integration_tests/tests/lib/database/database.g.dart index 3f6477c3..51eba060 100644 --- a/extras/integration_tests/tests/lib/database/database.g.dart +++ b/extras/integration_tests/tests/lib/database/database.g.dart @@ -147,6 +147,14 @@ class UsersCompanion extends UpdateCompanion { this.profilePicture = const Value.absent(), this.preferences = const Value.absent(), }); + UsersCompanion.insert({ + this.id = const Value.absent(), + @required String name, + @required DateTime birthDate, + this.profilePicture = const Value.absent(), + this.preferences = const Value.absent(), + }) : name = Value(name), + birthDate = Value(birthDate); UsersCompanion copyWith( {Value id, Value name, @@ -172,7 +180,8 @@ class $UsersTable extends Users with TableInfo<$UsersTable, User> { @override GeneratedIntColumn get id => _id ??= _constructId(); GeneratedIntColumn _constructId() { - return GeneratedIntColumn('id', $tableName, false, hasAutoIncrement: true); + return GeneratedIntColumn('id', $tableName, false, + hasAutoIncrement: true, declaredAsPrimaryKey: true); } final VerificationMeta _nameMeta = const VerificationMeta('name'); @@ -402,6 +411,12 @@ class FriendshipsCompanion extends UpdateCompanion { this.secondUser = const Value.absent(), this.reallyGoodFriends = const Value.absent(), }); + FriendshipsCompanion.insert({ + @required int firstUser, + @required int secondUser, + this.reallyGoodFriends = const Value.absent(), + }) : firstUser = Value(firstUser), + secondUser = Value(secondUser); FriendshipsCompanion copyWith( {Value firstUser, Value secondUser, @@ -520,6 +535,165 @@ class $FriendshipsTable extends Friendships } } +abstract class _$Database extends GeneratedDatabase { + _$Database(QueryExecutor e) : super(const SqlTypeSystem.withDefaults(), e); + $UsersTable _users; + $UsersTable get users => _users ??= $UsersTable(this); + $FriendshipsTable _friendships; + $FriendshipsTable get friendships => _friendships ??= $FriendshipsTable(this); + User _rowToUser(QueryRow row) { + return User( + id: row.readInt('id'), + name: row.readString('name'), + birthDate: row.readDateTime('birth_date'), + profilePicture: row.readBlob('profile_picture'), + preferences: + $UsersTable.$converter0.mapToDart(row.readString('preferences')), + ); + } + + Selectable mostPopularUsersQuery( + int amount, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'SELECT * FROM users u ORDER BY (SELECT COUNT(*) FROM friendships WHERE first_user = u.id OR second_user = u.id) DESC LIMIT :amount', + variables: [ + Variable.withInt(amount), + ], + readsFrom: { + users, + friendships + }).map(_rowToUser); + } + + Future> mostPopularUsers( + int amount, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return mostPopularUsersQuery(amount, operateOn: operateOn).get(); + } + + Stream> watchMostPopularUsers(int amount) { + return mostPopularUsersQuery(amount).watch(); + } + + AmountOfGoodFriendsResult _rowToAmountOfGoodFriendsResult(QueryRow row) { + return AmountOfGoodFriendsResult( + count: row.readInt('COUNT(*)'), + ); + } + + Selectable amountOfGoodFriendsQuery( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'SELECT COUNT(*) FROM friendships f WHERE f.really_good_friends AND (f.first_user = :user OR f.second_user = :user)', + variables: [ + Variable.withInt(user), + ], + readsFrom: { + friendships + }).map(_rowToAmountOfGoodFriendsResult); + } + + Future> amountOfGoodFriends( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return amountOfGoodFriendsQuery(user, operateOn: operateOn).get(); + } + + Stream> watchAmountOfGoodFriends(int user) { + return amountOfGoodFriendsQuery(user).watch(); + } + + Selectable friendsOfQuery( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'SELECT u.* FROM friendships f\n INNER JOIN users u ON u.id IN (f.first_user, f.second_user) AND\n u.id != :user\n WHERE (f.first_user = :user OR f.second_user = :user)', + variables: [ + Variable.withInt(user), + ], + readsFrom: { + friendships, + users + }).map(_rowToUser); + } + + Future> friendsOf( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return friendsOfQuery(user, operateOn: operateOn).get(); + } + + Stream> watchFriendsOf(int user) { + return friendsOfQuery(user).watch(); + } + + UserCountResult _rowToUserCountResult(QueryRow row) { + return UserCountResult( + cOUNTid: row.readInt('COUNT(id)'), + ); + } + + Selectable userCountQuery( + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery('SELECT COUNT(id) FROM users', + variables: [], readsFrom: {users}).map(_rowToUserCountResult); + } + + Future> userCount( + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return userCountQuery(operateOn: operateOn).get(); + } + + Stream> watchUserCount() { + return userCountQuery().watch(); + } + + SettingsForResult _rowToSettingsForResult(QueryRow row) { + return SettingsForResult( + preferences: + $UsersTable.$converter0.mapToDart(row.readString('preferences')), + ); + } + + Selectable settingsForQuery( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'SELECT preferences FROM users WHERE id = :user', + variables: [ + Variable.withInt(user), + ], + readsFrom: { + users + }).map(_rowToSettingsForResult); + } + + Future> settingsFor( + int user, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return settingsForQuery(user, operateOn: operateOn).get(); + } + + Stream> watchSettingsFor(int user) { + return settingsForQuery(user).watch(); + } + + @override + List get allTables => [users, friendships]; +} + class AmountOfGoodFriendsResult { final int count; AmountOfGoodFriendsResult({ @@ -540,145 +714,3 @@ class SettingsForResult { this.preferences, }); } - -abstract class _$Database extends GeneratedDatabase { - _$Database(QueryExecutor e) : super(const SqlTypeSystem.withDefaults(), e); - $UsersTable _users; - $UsersTable get users => _users ??= $UsersTable(this); - $FriendshipsTable _friendships; - $FriendshipsTable get friendships => _friendships ??= $FriendshipsTable(this); - User _rowToUser(QueryRow row) { - return User( - id: row.readInt('id'), - name: row.readString('name'), - birthDate: row.readDateTime('birth_date'), - profilePicture: row.readBlob('profile_picture'), - preferences: - $UsersTable.$converter0.mapToDart(row.readString('preferences')), - ); - } - - Future> mostPopularUsers( - int amount, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT * FROM users u ORDER BY (SELECT COUNT(*) FROM friendships WHERE first_user = u.id OR second_user = u.id) DESC LIMIT :amount', - variables: [ - Variable.withInt(amount), - ]).then((rows) => rows.map(_rowToUser).toList()); - } - - Stream> watchMostPopularUsers(int amount) { - return customSelectStream( - 'SELECT * FROM users u ORDER BY (SELECT COUNT(*) FROM friendships WHERE first_user = u.id OR second_user = u.id) DESC LIMIT :amount', - variables: [ - Variable.withInt(amount), - ], - readsFrom: { - users, - friendships - }).map((rows) => rows.map(_rowToUser).toList()); - } - - AmountOfGoodFriendsResult _rowToAmountOfGoodFriendsResult(QueryRow row) { - return AmountOfGoodFriendsResult( - count: row.readInt('COUNT(*)'), - ); - } - - Future> amountOfGoodFriends( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT COUNT(*) FROM friendships f WHERE f.really_good_friends AND (f.first_user = :user OR f.second_user = :user)', - variables: [ - Variable.withInt(user), - ]).then((rows) => rows.map(_rowToAmountOfGoodFriendsResult).toList()); - } - - Stream> watchAmountOfGoodFriends(int user) { - return customSelectStream( - 'SELECT COUNT(*) FROM friendships f WHERE f.really_good_friends AND (f.first_user = :user OR f.second_user = :user)', - variables: [ - Variable.withInt(user), - ], - readsFrom: { - friendships - }).map((rows) => rows.map(_rowToAmountOfGoodFriendsResult).toList()); - } - - Future> friendsOf( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT u.* FROM friendships f\n INNER JOIN users u ON u.id IN (f.first_user, f.second_user) AND\n u.id != :user\n WHERE (f.first_user = :user OR f.second_user = :user)', - variables: [ - Variable.withInt(user), - ]).then((rows) => rows.map(_rowToUser).toList()); - } - - Stream> watchFriendsOf(int user) { - return customSelectStream( - 'SELECT u.* FROM friendships f\n INNER JOIN users u ON u.id IN (f.first_user, f.second_user) AND\n u.id != :user\n WHERE (f.first_user = :user OR f.second_user = :user)', - variables: [ - Variable.withInt(user), - ], - readsFrom: { - friendships, - users - }).map((rows) => rows.map(_rowToUser).toList()); - } - - UserCountResult _rowToUserCountResult(QueryRow row) { - return UserCountResult( - cOUNTid: row.readInt('COUNT(id)'), - ); - } - - Future> userCount( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelect('SELECT COUNT(id) FROM users', - variables: []).then((rows) => rows.map(_rowToUserCountResult).toList()); - } - - Stream> watchUserCount() { - return customSelectStream('SELECT COUNT(id) FROM users', - variables: [], readsFrom: {users}) - .map((rows) => rows.map(_rowToUserCountResult).toList()); - } - - SettingsForResult _rowToSettingsForResult(QueryRow row) { - return SettingsForResult( - preferences: - $UsersTable.$converter0.mapToDart(row.readString('preferences')), - ); - } - - Future> settingsFor( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelect( - 'SELECT preferences FROM users WHERE id = :user', - variables: [ - Variable.withInt(user), - ]).then((rows) => rows.map(_rowToSettingsForResult).toList()); - } - - Stream> watchSettingsFor(int user) { - return customSelectStream('SELECT preferences FROM users WHERE id = :user', - variables: [ - Variable.withInt(user), - ], - readsFrom: { - users - }).map((rows) => rows.map(_rowToSettingsForResult).toList()); - } - - @override - List get allTables => [users, friendships]; -} diff --git a/moor_generator/lib/src/backends/build/build_backend.dart b/moor_generator/lib/src/backends/build/build_backend.dart index 8867e961..6fa105ea 100644 --- a/moor_generator/lib/src/backends/build/build_backend.dart +++ b/moor_generator/lib/src/backends/build/build_backend.dart @@ -1,4 +1,3 @@ -import 'package:analyzer/analyzer.dart'; import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart' hide log; diff --git a/sqlparser/lib/src/ast/moor/import_statement.dart b/sqlparser/lib/src/ast/moor/import_statement.dart index 1fa6863f..31115896 100644 --- a/sqlparser/lib/src/ast/moor/import_statement.dart +++ b/sqlparser/lib/src/ast/moor/import_statement.dart @@ -8,7 +8,7 @@ class ImportStatement extends Statement { ImportStatement(this.importedFile); @override - T accept(AstVisitor visitor) {} + T accept(AstVisitor visitor) => visitor.visitMoorImportStatement(this); @override final Iterable childNodes = const []; From 1ed1e3ba94839037678c9dc9f1889bcb94263d68 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 20:26:51 +0200 Subject: [PATCH 058/117] Parse declared statements in moor files --- sqlparser/lib/src/ast/ast.dart | 5 +++ .../lib/src/ast/moor/declared_statement.dart | 25 +++++++++++++ .../lib/src/ast/moor/import_statement.dart | 1 + sqlparser/lib/src/reader/parser/parser.dart | 37 ++++++++++++++++--- .../test/parser/multiple_statements.dart | 16 +++++++- 5 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 sqlparser/lib/src/ast/moor/declared_statement.dart diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index 403ac59d..bd85fdd5 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -21,6 +21,7 @@ part 'expressions/subquery.dart'; part 'expressions/tuple.dart'; part 'expressions/variables.dart'; +part 'moor/declared_statement.dart'; part 'moor/import_statement.dart'; part 'schema/column_definition.dart'; @@ -178,6 +179,7 @@ abstract class AstVisitor { T visitNamedVariable(ColonNamedVariable e); T visitMoorImportStatement(ImportStatement e); + T visitMoorDeclaredStatement(DeclaredStatement e); } /// Visitor that walks down the entire tree, visiting all children in order. @@ -293,6 +295,9 @@ class RecursiveVisitor extends AstVisitor { @override T visitMoorImportStatement(ImportStatement e) => visitChildren(e); + @override + T visitMoorDeclaredStatement(DeclaredStatement e) => visitChildren(e); + @protected T visitChildren(AstNode e) { for (var child in e.childNodes) { diff --git a/sqlparser/lib/src/ast/moor/declared_statement.dart b/sqlparser/lib/src/ast/moor/declared_statement.dart new file mode 100644 index 00000000..f4555975 --- /dev/null +++ b/sqlparser/lib/src/ast/moor/declared_statement.dart @@ -0,0 +1,25 @@ +part of '../ast.dart'; + +/// A declared statement inside a `.moor` file. It consists of an identifier, +/// followed by a colon and the query to run. +class DeclaredStatement extends Statement { + final String name; + final CrudStatement statement; + + IdentifierToken identifier; + Token colon; + + DeclaredStatement(this.name, this.statement); + + @override + T accept(AstVisitor visitor) => + visitor.visitMoorDeclaredStatement(this); + + @override + Iterable get childNodes => [statement]; + + @override + bool contentEquals(DeclaredStatement other) { + return other.name == name; + } +} diff --git a/sqlparser/lib/src/ast/moor/import_statement.dart b/sqlparser/lib/src/ast/moor/import_statement.dart index 31115896..b542565c 100644 --- a/sqlparser/lib/src/ast/moor/import_statement.dart +++ b/sqlparser/lib/src/ast/moor/import_statement.dart @@ -1,5 +1,6 @@ part of '../ast.dart'; +/// An `import "file.dart";` statement that can appear inside a moor file. class ImportStatement extends Statement { Token importToken; StringLiteralToken importString; diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 340ee8bf..8af014bb 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -157,14 +157,11 @@ class Parser extends ParserBase Statement statement({bool expectEnd = true}) { final first = _peek; - var stmt = select() ?? - _deleteStmt() ?? - _update() ?? - _insertStmt() ?? - _createTable(); + Statement stmt = _crud(); + stmt ??= _createTable(); if (enableMoorExtensions) { - stmt ??= _import(); + stmt ??= _import() ?? _declaredStatement(); } if (stmt == null) { @@ -181,6 +178,17 @@ class Parser extends ParserBase return stmt..setSpan(first, _previous); } + CrudStatement _crud() { + // writing select() ?? _deleteStmt() and so on doesn't cast to CrudStatement + // for some reason. + CrudStatement stmt = select(); + stmt ??= _deleteStmt(); + stmt ??= _update(); + stmt ??= _insertStmt(); + + return stmt; + } + ImportStatement _import() { if (_matchOne(TokenType.import)) { final importToken = _previous; @@ -195,6 +203,23 @@ class Parser extends ParserBase return null; } + DeclaredStatement _declaredStatement() { + if (_check(TokenType.identifier) || _peek is KeywordToken) { + final name = _consumeIdentifier('Expected a name for a declared query', + lenient: true); + final colon = + _consume(TokenType.colon, 'Expected colon (:) followed by a query'); + + final stmt = _crud(); + + return DeclaredStatement(name.identifier, stmt) + ..identifier = name + ..colon = colon; + } + + return null; + } + List statements() { final stmts = []; while (!_isAtEnd) { diff --git a/sqlparser/test/parser/multiple_statements.dart b/sqlparser/test/parser/multiple_statements.dart index 43b0e1f7..943a717c 100644 --- a/sqlparser/test/parser/multiple_statements.dart +++ b/sqlparser/test/parser/multiple_statements.dart @@ -46,10 +46,10 @@ void main() { ); }); - test('parses import directives in moor mode', () { + test('parses imports and declared statements in moor mode', () { final sql = r''' import 'test.dart'; - SELECT * FROM tbl; + query: SELECT * FROM tbl; '''; final tokens = Scanner(sql, scanMoorTokens: true).scanTokens(); @@ -62,5 +62,17 @@ void main() { expect(parsedImport.importToken, tokens[0]); expect(parsedImport.importString, tokens[1]); expect(parsedImport.semicolon, tokens[2]); + + final declared = statements[1] as DeclaredStatement; + enforceEqual( + declared, + DeclaredStatement( + 'query', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + ), + ), + ); }); } From 35e5bdb3170a7204251099c5ec4a3eb32d39a0ce Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 21:33:56 +0200 Subject: [PATCH 059/117] Introduce new top-level ast node for moor files --- .../analyzer/moor/create_table_reader.dart | 7 +- .../lib/src/analyzer/moor/parser.dart | 23 +++---- sqlparser/lib/src/ast/ast.dart | 5 ++ .../lib/src/ast/moor/declared_statement.dart | 2 +- .../lib/src/ast/moor/import_statement.dart | 2 +- sqlparser/lib/src/ast/moor/moor_file.dart | 19 ++++++ .../lib/src/ast/statements/create_table.dart | 4 +- .../descriptions/description.dart | 3 + .../lib/src/engine/autocomplete/engine.dart | 24 +++++++ sqlparser/lib/src/engine/sql_engine.dart | 21 +++--- sqlparser/lib/src/reader/parser/parser.dart | 67 +++++++++++++++---- .../test/parser/multiple_statements.dart | 49 ++++++++------ 12 files changed, 161 insertions(+), 65 deletions(-) create mode 100644 sqlparser/lib/src/ast/moor/moor_file.dart create mode 100644 sqlparser/lib/src/engine/autocomplete/descriptions/description.dart create mode 100644 sqlparser/lib/src/engine/autocomplete/engine.dart diff --git a/moor_generator/lib/src/analyzer/moor/create_table_reader.dart b/moor_generator/lib/src/analyzer/moor/create_table_reader.dart index 21bf136d..e9381760 100644 --- a/moor_generator/lib/src/analyzer/moor/create_table_reader.dart +++ b/moor_generator/lib/src/analyzer/moor/create_table_reader.dart @@ -9,13 +9,12 @@ import 'package:sqlparser/sqlparser.dart'; class CreateTableReader { /// The AST of this `CREATE TABLE` statement. - final ParseResult ast; + final CreateTableStatement stmt; - CreateTableReader(this.ast); + CreateTableReader(this.stmt); SpecifiedTable extractTable(TypeMapper mapper) { - final table = - SchemaFromCreateTable().read(ast.rootNode as CreateTableStatement); + final table = SchemaFromCreateTable().read(stmt); final foundColumns = {}; final primaryKey = {}; diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index 05a7d927..316ba9f9 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -10,29 +10,27 @@ class MoorParser { MoorParser(this.task); Future parseAndAnalyze() { - final results = - SqlEngine(useMoorExtensions: true).parseMultiple(task.content); + final result = + SqlEngine(useMoorExtensions: true).parseMoorFile(task.content); + final parsedFile = result.rootNode as MoorFile; final createdReaders = []; - for (var parsedStmt in results) { - if (parsedStmt.rootNode is ImportStatement) { - final importStmt = (parsedStmt.rootNode) as ImportStatement; + for (var parsedStmt in parsedFile.statements) { + if (parsedStmt is ImportStatement) { + final importStmt = parsedStmt; task.inlineDartResolver.importStatements.add(importStmt.importedFile); - } else if (parsedStmt.rootNode is CreateTableStatement) { + } else if (parsedStmt is CreateTableStatement) { createdReaders.add(CreateTableReader(parsedStmt)); } else { task.reportError(ErrorInMoorFile( - span: parsedStmt.rootNode.span, + span: parsedStmt.span, message: 'At the moment, only CREATE TABLE statements are supported' 'in .moor files')); } } - // all results have the same list of errors - final sqlErrors = results.isEmpty ? [] : results.first.errors; - - for (var error in sqlErrors) { + for (var error in result.errors) { task.reportError(ErrorInMoorFile( span: error.token.span, message: error.message, @@ -41,8 +39,7 @@ class MoorParser { final createdTables = createdReaders.map((r) => r.extractTable(task.mapper)).toList(); - final parsedFile = ParsedMoorFile(createdTables); - return Future.value(parsedFile); + return Future.value(ParsedMoorFile(createdTables)); } } diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index bd85fdd5..2734145b 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -23,6 +23,7 @@ part 'expressions/variables.dart'; part 'moor/declared_statement.dart'; part 'moor/import_statement.dart'; +part 'moor/moor_file.dart'; part 'schema/column_definition.dart'; part 'schema/table_definition.dart'; @@ -178,6 +179,7 @@ abstract class AstVisitor { T visitNumberedVariable(NumberedVariable e); T visitNamedVariable(ColonNamedVariable e); + T visitMoorFile(MoorFile e); T visitMoorImportStatement(ImportStatement e); T visitMoorDeclaredStatement(DeclaredStatement e); } @@ -292,6 +294,9 @@ class RecursiveVisitor extends AstVisitor { @override T visitFrameSpec(FrameSpec e) => visitChildren(e); + @override + T visitMoorFile(MoorFile e) => visitChildren(e); + @override T visitMoorImportStatement(ImportStatement e) => visitChildren(e); diff --git a/sqlparser/lib/src/ast/moor/declared_statement.dart b/sqlparser/lib/src/ast/moor/declared_statement.dart index f4555975..bb73cae8 100644 --- a/sqlparser/lib/src/ast/moor/declared_statement.dart +++ b/sqlparser/lib/src/ast/moor/declared_statement.dart @@ -2,7 +2,7 @@ part of '../ast.dart'; /// A declared statement inside a `.moor` file. It consists of an identifier, /// followed by a colon and the query to run. -class DeclaredStatement extends Statement { +class DeclaredStatement extends Statement implements PartOfMoorFile { final String name; final CrudStatement statement; diff --git a/sqlparser/lib/src/ast/moor/import_statement.dart b/sqlparser/lib/src/ast/moor/import_statement.dart index b542565c..bdbfc8d8 100644 --- a/sqlparser/lib/src/ast/moor/import_statement.dart +++ b/sqlparser/lib/src/ast/moor/import_statement.dart @@ -1,7 +1,7 @@ part of '../ast.dart'; /// An `import "file.dart";` statement that can appear inside a moor file. -class ImportStatement extends Statement { +class ImportStatement extends Statement implements PartOfMoorFile { Token importToken; StringLiteralToken importString; final String importedFile; diff --git a/sqlparser/lib/src/ast/moor/moor_file.dart b/sqlparser/lib/src/ast/moor/moor_file.dart new file mode 100644 index 00000000..d451d14f --- /dev/null +++ b/sqlparser/lib/src/ast/moor/moor_file.dart @@ -0,0 +1,19 @@ +part of '../ast.dart'; + +/// Something that can appear as a top-level declaration inside a `.moor` file. +abstract class PartOfMoorFile implements Statement {} + +class MoorFile extends AstNode { + final List statements; + + MoorFile(this.statements); + + @override + T accept(AstVisitor visitor) => visitor.visitMoorFile(this); + + @override + Iterable get childNodes => statements; + + @override + bool contentEquals(MoorFile other) => true; +} diff --git a/sqlparser/lib/src/ast/statements/create_table.dart b/sqlparser/lib/src/ast/statements/create_table.dart index ee000b28..060a1bc3 100644 --- a/sqlparser/lib/src/ast/statements/create_table.dart +++ b/sqlparser/lib/src/ast/statements/create_table.dart @@ -2,7 +2,9 @@ part of '../ast.dart'; /// A "CREATE TABLE" statement, see https://www.sqlite.org/lang_createtable.html /// for the individual components. -class CreateTableStatement extends Statement with SchemaStatement { +class CreateTableStatement extends Statement + with SchemaStatement + implements PartOfMoorFile { final bool ifNotExists; final String tableName; final List columns; diff --git a/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart b/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart new file mode 100644 index 00000000..6f8cc854 --- /dev/null +++ b/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart @@ -0,0 +1,3 @@ +part of '../engine.dart'; + +abstract class HintDescription {} diff --git a/sqlparser/lib/src/engine/autocomplete/engine.dart b/sqlparser/lib/src/engine/autocomplete/engine.dart new file mode 100644 index 00000000..df47cdad --- /dev/null +++ b/sqlparser/lib/src/engine/autocomplete/engine.dart @@ -0,0 +1,24 @@ +import 'package:sqlparser/src/reader/tokenizer/token.dart'; + +part 'descriptions/description.dart'; + +/// Helper to provide context aware auto-complete suggestions inside a sql +/// query. +/// +/// While parsing a query, the parser will yield a bunch of [Hint]s that are +/// specific to a specific location. Each hint contains the current position and +/// a [HintDescription] of what can appear behind that position. +/// To obtain suggestions for a specific cursor position, we then go back from +/// that position to the last [Hint] found and populate it. +class AutoCompleteEngine { + final List foundHints = []; +} + +class Hint { + /// The token that appears just before this hint, or `null` if the hint + /// appears at the beginning of the file. + final Token before; + final HintDescription description; + + Hint(this.before, this.description); +} diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index f8061a38..3d177326 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -52,22 +52,17 @@ class SqlEngine { return ParseResult._(stmt, parser.errors, sql); } - /// Parses multiple sql statements, separated by a semicolon. All - /// [ParseResult] entries will have the same [ParseResult.errors], but the - /// [ParseResult.sql] will only refer to the substring creating a statement. - List parseMultiple(String sql) { - final tokens = tokenize(sql); - final parser = Parser(tokens); + /// Parses a `.moor` file, which can consist of multiple statements and + /// additional components like import statements. + ParseResult parseMoorFile(String content) { + assert(useMoorExtensions); - final stmts = parser.statements(); + final tokens = tokenize(content); + final parser = Parser(tokens, useMoor: true); - return stmts.map((statement) { - final first = statement.firstPosition; - final last = statement.lastPosition; + final moorFile = parser.moorFile(); - final source = sql.substring(first, last); - return ParseResult._(statement, parser.errors, source); - }).toList(); + return ParseResult._(moorFile, parser.errors, content); } /// Parses and analyzes the [sql] statement. The [AnalysisContext] returned diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 8af014bb..eb0b24c2 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -155,7 +155,7 @@ class Parser extends ParserBase with ExpressionParser, SchemaParser, CrudParser { Parser(List tokens, {bool useMoor = false}) : super(tokens, useMoor); - Statement statement({bool expectEnd = true}) { + Statement statement() { final first = _peek; Statement stmt = _crud(); stmt ??= _createTable(); @@ -172,7 +172,7 @@ class Parser extends ParserBase stmt.semicolon = _previous; } - if (!_isAtEnd && expectEnd) { + if (!_isAtEnd) { _error('Expected the statement to finish here'); } return stmt..setSpan(first, _previous); @@ -189,6 +189,38 @@ class Parser extends ParserBase return stmt; } + MoorFile moorFile() { + final first = _peek; + final foundComponents = []; + + // first, parse import statements + for (var stmt = _parseAsStatement(_import); + stmt != null; + stmt = _parseAsStatement(_import)) { + foundComponents.add(stmt); + } + + // next, table declarations + for (var stmt = _parseAsStatement(_createTable); + stmt != null; + stmt = _parseAsStatement(_createTable)) { + foundComponents.add(stmt); + } + + // finally, declared statements + for (var stmt = _parseAsStatement(_declaredStatement); + stmt != null; + stmt = _parseAsStatement(_declaredStatement)) { + foundComponents.add(stmt); + } + + if (!_isAtEnd) { + _error('Expected the file to end here.'); + } + + return MoorFile(foundComponents)..setSpan(first, _previous); + } + ImportStatement _import() { if (_matchOne(TokenType.import)) { final importToken = _previous; @@ -220,18 +252,27 @@ class Parser extends ParserBase return null; } - List statements() { - final stmts = []; - while (!_isAtEnd) { - try { - stmts.add(statement(expectEnd: false)); - } on ParsingError catch (_) { - // the error is added to the list errors, so ignore. We skip to the next - // semicolon to parse the next statement. - _synchronize(); - } + /// Invokes [parser], sets the appropriate source span and attaches a + /// semicolon if one exists. + T _parseAsStatement(T Function() parser) { + final first = _peek; + T result; + try { + result = parser(); + } on ParsingError catch (_) { + // the error is added to the list errors, so ignore. We skip to the next + // semicolon to parse the next statement. + _synchronize(); } - return stmts; + + if (result == null) return null; + + if (_matchOne(TokenType.semicolon)) { + result.semicolon = _previous; + } + + result.setSpan(first, _previous); + return result; } void _synchronize() { diff --git a/sqlparser/test/parser/multiple_statements.dart b/sqlparser/test/parser/multiple_statements.dart index 943a717c..0842b81b 100644 --- a/sqlparser/test/parser/multiple_statements.dart +++ b/sqlparser/test/parser/multiple_statements.dart @@ -6,42 +6,53 @@ import 'package:test/test.dart'; void main() { test('can parse multiple statements', () { - final sql = 'UPDATE tbl SET a = b; SELECT * FROM tbl;'; + final sql = 'a: UPDATE tbl SET a = b; b: SELECT * FROM tbl;'; final tokens = Scanner(sql).scanTokens(); - final statements = Parser(tokens).statements(); + final moorFile = Parser(tokens).moorFile(); + + final statements = moorFile.statements; enforceEqual( statements[0], - UpdateStatement( - table: TableReference('tbl', null), - set: [ - SetComponent( - column: Reference(columnName: 'a'), - expression: Reference(columnName: 'b'), - ), - ], + DeclaredStatement( + 'a', + UpdateStatement( + table: TableReference('tbl', null), + set: [ + SetComponent( + column: Reference(columnName: 'a'), + expression: Reference(columnName: 'b'), + ), + ], + ), ), ); enforceEqual( statements[1], - SelectStatement( - columns: [StarResultColumn(null)], - from: [TableReference('tbl', null)], + DeclaredStatement( + 'b', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + ), ), ); }); test('recovers from invalid statements', () { - final sql = 'UPDATE tbl SET a = * d; SELECT * FROM tbl;'; + final sql = 'a: UPDATE tbl SET a = * d; b: SELECT * FROM tbl;'; final tokens = Scanner(sql).scanTokens(); - final statements = Parser(tokens).statements(); + final statements = Parser(tokens).moorFile().statements; expect(statements, hasLength(1)); enforceEqual( statements[0], - SelectStatement( - columns: [StarResultColumn(null)], - from: [TableReference('tbl', null)], + DeclaredStatement( + 'b', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + ), ), ); }); @@ -53,7 +64,7 @@ void main() { '''; final tokens = Scanner(sql, scanMoorTokens: true).scanTokens(); - final statements = Parser(tokens, useMoor: true).statements(); + final statements = Parser(tokens, useMoor: true).moorFile().statements; expect(statements, hasLength(2)); From ee9b413e5d85a74d2f0df77d7d01983a0e67737b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 22:18:45 +0200 Subject: [PATCH 060/117] Start experimenting with auto-completion features --- .../descriptions/description.dart | 13 ++++- .../autocomplete/descriptions/static.dart | 20 +++++++ .../lib/src/engine/autocomplete/engine.dart | 56 ++++++++++++++++++- .../src/engine/autocomplete/suggestion.dart | 30 ++++++++++ sqlparser/lib/src/engine/sql_engine.dart | 12 +++- sqlparser/lib/src/reader/parser/parser.dart | 12 +++- sqlparser/lib/src/reader/parser/schema.dart | 3 + sqlparser/lib/src/reader/tokenizer/token.dart | 6 ++ 8 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 sqlparser/lib/src/engine/autocomplete/descriptions/static.dart create mode 100644 sqlparser/lib/src/engine/autocomplete/suggestion.dart diff --git a/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart b/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart index 6f8cc854..cef121de 100644 --- a/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart +++ b/sqlparser/lib/src/engine/autocomplete/descriptions/description.dart @@ -1,3 +1,14 @@ part of '../engine.dart'; -abstract class HintDescription {} +/// Attached to a hint. A [HintDescription], together with additional context, +/// can be used to compute the available autocomplete suggestions. +abstract class HintDescription { + const HintDescription(); + + const factory HintDescription.tokens(List types) = + TokensDescription; + + factory HintDescription.token(TokenType type) = TokensDescription.single; + + Iterable suggest(CalculationRequest request); +} diff --git a/sqlparser/lib/src/engine/autocomplete/descriptions/static.dart b/sqlparser/lib/src/engine/autocomplete/descriptions/static.dart new file mode 100644 index 00000000..8587eae6 --- /dev/null +++ b/sqlparser/lib/src/engine/autocomplete/descriptions/static.dart @@ -0,0 +1,20 @@ +part of '../engine.dart'; + +/// Suggestion that just inserts a bunch of token types with whitespace in +/// between. +class TokensDescription extends HintDescription { + final List types; + + const TokensDescription(this.types); + TokensDescription.single(TokenType type) : types = [type]; + + @override + Iterable suggest(CalculationRequest request) sync* { + final code = types + .map((type) => reverseKeywords[type]) + .where((k) => k != null) + .join(' '); + + yield Suggestion(code, 0); + } +} diff --git a/sqlparser/lib/src/engine/autocomplete/engine.dart b/sqlparser/lib/src/engine/autocomplete/engine.dart index df47cdad..6914931e 100644 --- a/sqlparser/lib/src/engine/autocomplete/engine.dart +++ b/sqlparser/lib/src/engine/autocomplete/engine.dart @@ -1,6 +1,10 @@ +import 'package:collection/collection.dart'; import 'package:sqlparser/src/reader/tokenizer/token.dart'; part 'descriptions/description.dart'; +part 'descriptions/static.dart'; + +part 'suggestion.dart'; /// Helper to provide context aware auto-complete suggestions inside a sql /// query. @@ -11,14 +15,64 @@ part 'descriptions/description.dart'; /// To obtain suggestions for a specific cursor position, we then go back from /// that position to the last [Hint] found and populate it. class AutoCompleteEngine { - final List foundHints = []; + /// The found hints. + UnmodifiableListView get foundHints => _hintsView; + // hints are always sorted by their offset + final List _hints = []; + UnmodifiableListView _hintsView; + + void addHint(Hint hint) { + _hints.insert(_lastHintBefore(hint.offset), hint); + } + + AutoCompleteEngine() { + _hintsView = UnmodifiableListView(_hints); + } + + /// Suggest completions at a specific position. + /// + /// This api will change in the future. + ComputedSuggestions suggestCompletions(int offset) { + final hint = foundHints[_lastHintBefore(offset)]; + + final suggestions = hint.description.suggest(CalculationRequest()).toList(); + return ComputedSuggestions(hint.offset, offset - hint.offset, suggestions); + } + + int _lastHintBefore(int offset) { + // find the last hint that appears before offset + var min = 0; + var max = foundHints.length; + + while (min < max) { + final mid = min + ((max - min) >> 1); + final hint = _hints[mid]; + + final offsetOfMid = hint.offset; + + if (offsetOfMid == offset) { + return mid; + } else if (offsetOfMid < offset) { + min = mid + 1; + } else { + max = mid - 1; + } + } + + return min; + } } class Hint { /// The token that appears just before this hint, or `null` if the hint /// appears at the beginning of the file. final Token before; + + int get offset => before.span.end.offset; + final HintDescription description; Hint(this.before, this.description); } + +class CalculationRequest {} diff --git a/sqlparser/lib/src/engine/autocomplete/suggestion.dart b/sqlparser/lib/src/engine/autocomplete/suggestion.dart new file mode 100644 index 00000000..8b92dc90 --- /dev/null +++ b/sqlparser/lib/src/engine/autocomplete/suggestion.dart @@ -0,0 +1,30 @@ +part of 'engine.dart'; + +/// The result of suggesting auto-complete at a specific location. +class ComputedSuggestions { + /// The offset from the source file from which the suggestion should be + /// applied. Effectively, the range from [anchor] to `anchor + lengthBefore` + /// will be replaced with the suggestion. + final int anchor; + + /// The amount of chars that have already been typed and would be replaced + /// when applying a suggestion. + final int lengthBefore; + + /// The actual suggestions which are relevant here. + final List suggestions; + + ComputedSuggestions(this.anchor, this.lengthBefore, this.suggestions); +} + +/// A single auto-complete suggestion. +class Suggestion { + /// The code inserted. + final String code; + + /// The relevance of this suggestion, where more relevant suggestions have a + /// higher [relevance]. + final int relevance; + + Suggestion(this.code, this.relevance); +} diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 3d177326..05477e5d 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -1,5 +1,6 @@ import 'package:sqlparser/src/analysis/analysis.dart'; import 'package:sqlparser/src/ast/ast.dart'; +import 'package:sqlparser/src/engine/autocomplete/engine.dart'; import 'package:sqlparser/src/reader/parser/parser.dart'; import 'package:sqlparser/src/reader/tokenizer/scanner.dart'; import 'package:sqlparser/src/reader/tokenizer/token.dart'; @@ -49,7 +50,7 @@ class SqlEngine { final parser = Parser(tokens, useMoor: useMoorExtensions); final stmt = parser.statement(); - return ParseResult._(stmt, parser.errors, sql); + return ParseResult._(stmt, parser.errors, sql, null); } /// Parses a `.moor` file, which can consist of multiple statements and @@ -57,12 +58,13 @@ class SqlEngine { ParseResult parseMoorFile(String content) { assert(useMoorExtensions); + final autoComplete = AutoCompleteEngine(); final tokens = tokenize(content); final parser = Parser(tokens, useMoor: true); final moorFile = parser.moorFile(); - return ParseResult._(moorFile, parser.errors, content); + return ParseResult._(moorFile, parser.errors, content, autoComplete); } /// Parses and analyzes the [sql] statement. The [AnalysisContext] returned @@ -121,5 +123,9 @@ class ParseResult { /// The sql source that created the AST at [rootNode]. final String sql; - ParseResult._(this.rootNode, this.errors, this.sql); + /// The engine which can be used to handle auto-complete requests on this + /// result. + final AutoCompleteEngine autoCompleteEngine; + + ParseResult._(this.rootNode, this.errors, this.sql, this.autoCompleteEngine); } diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index eb0b24c2..700d7337 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -1,5 +1,6 @@ import 'package:meta/meta.dart'; import 'package:sqlparser/src/ast/ast.dart'; +import 'package:sqlparser/src/engine/autocomplete/engine.dart'; import 'package:sqlparser/src/reader/tokenizer/token.dart'; part 'crud.dart'; @@ -43,13 +44,18 @@ class ParsingError implements Exception { abstract class ParserBase { final List tokens; final List errors = []; + final AutoCompleteEngine autoComplete; /// Whether to enable the extensions moor makes to the sql grammar. final bool enableMoorExtensions; int _current = 0; - ParserBase(this.tokens, this.enableMoorExtensions); + ParserBase(this.tokens, this.enableMoorExtensions, this.autoComplete); + + void _suggestHint(HintDescription description) { + autoComplete?.addHint(Hint(_previous, description)); + } bool get _isAtEnd => _peek.type == TokenType.eof; Token get _peek => tokens[_current]; @@ -153,7 +159,9 @@ abstract class ParserBase { class Parser extends ParserBase with ExpressionParser, SchemaParser, CrudParser { - Parser(List tokens, {bool useMoor = false}) : super(tokens, useMoor); + Parser(List tokens, + {bool useMoor = false, AutoCompleteEngine autoComplete}) + : super(tokens, useMoor, autoComplete); Statement statement() { final first = _peek; diff --git a/sqlparser/lib/src/reader/parser/schema.dart b/sqlparser/lib/src/reader/parser/schema.dart index fc4c7551..6b5258ad 100644 --- a/sqlparser/lib/src/reader/parser/schema.dart +++ b/sqlparser/lib/src/reader/parser/schema.dart @@ -2,9 +2,12 @@ part of 'parser.dart'; mixin SchemaParser on ParserBase { CreateTableStatement _createTable() { + _suggestHint( + const HintDescription.tokens([TokenType.create, TokenType.table])); if (!_matchOne(TokenType.create)) return null; final first = _previous; + _suggestHint(HintDescription.token(TokenType.table)); _consume(TokenType.table, 'Expected TABLE keyword here'); var ifNotExists = false; diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index c7ea7f00..8f1a71fb 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -237,6 +237,12 @@ const Map keywords = { 'VALUES': TokenType.$values, }; +/// Maps [TokenType]s which are keywords to their lexeme. +final reverseKeywords = { + for (var entry in keywords.entries) entry.value: entry.key, + for (var entry in moorKeywords.entries) entry.value: entry.key, +}; + const Map moorKeywords = { 'MAPPED': TokenType.mapped, 'IMPORT': TokenType.import, From a62c076c4c3b431de2f898a631e8ddd925184dbc Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 7 Sep 2019 22:49:23 +0200 Subject: [PATCH 061/117] Suggest CREATE TABLE statements in moor files --- .../lib/src/engine/autocomplete/engine.dart | 6 ++- sqlparser/lib/src/engine/sql_engine.dart | 2 +- sqlparser/lib/src/reader/parser/parser.dart | 11 ++++- .../test/engine/autocomplete/static_test.dart | 43 +++++++++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 sqlparser/test/engine/autocomplete/static_test.dart diff --git a/sqlparser/lib/src/engine/autocomplete/engine.dart b/sqlparser/lib/src/engine/autocomplete/engine.dart index 6914931e..f8335ff4 100644 --- a/sqlparser/lib/src/engine/autocomplete/engine.dart +++ b/sqlparser/lib/src/engine/autocomplete/engine.dart @@ -33,6 +33,10 @@ class AutoCompleteEngine { /// /// This api will change in the future. ComputedSuggestions suggestCompletions(int offset) { + if (_hints.isEmpty) { + return ComputedSuggestions(-1, -1, []); + } + final hint = foundHints[_lastHintBefore(offset)]; final suggestions = hint.description.suggest(CalculationRequest()).toList(); @@ -68,7 +72,7 @@ class Hint { /// appears at the beginning of the file. final Token before; - int get offset => before.span.end.offset; + int get offset => before?.span?.end?.offset ?? 0; final HintDescription description; diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 05477e5d..cfeff401 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -60,7 +60,7 @@ class SqlEngine { final autoComplete = AutoCompleteEngine(); final tokens = tokenize(content); - final parser = Parser(tokens, useMoor: true); + final parser = Parser(tokens, useMoor: true, autoComplete: autoComplete); final moorFile = parser.moorFile(); diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 700d7337..007392a3 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -54,7 +54,8 @@ abstract class ParserBase { ParserBase(this.tokens, this.enableMoorExtensions, this.autoComplete); void _suggestHint(HintDescription description) { - autoComplete?.addHint(Hint(_previous, description)); + final tokenBefore = _current == 0 ? null : _previous; + autoComplete?.addHint(Hint(tokenBefore, description)); } bool get _isAtEnd => _peek.type == TokenType.eof; @@ -226,7 +227,13 @@ class Parser extends ParserBase _error('Expected the file to end here.'); } - return MoorFile(foundComponents)..setSpan(first, _previous); + final file = MoorFile(foundComponents); + if (foundComponents.isNotEmpty) { + file.setSpan(first, _previous); + } else { + file.setSpan(first, first); // empty file + } + return file; } ImportStatement _import() { diff --git a/sqlparser/test/engine/autocomplete/static_test.dart b/sqlparser/test/engine/autocomplete/static_test.dart new file mode 100644 index 00000000..9c347f43 --- /dev/null +++ b/sqlparser/test/engine/autocomplete/static_test.dart @@ -0,0 +1,43 @@ +import 'package:sqlparser/sqlparser.dart'; +import 'package:sqlparser/src/engine/autocomplete/engine.dart'; +import 'package:test/test.dart'; + +void main() { + test('suggests a CREATE TABLE statements for an empty file', () { + final engine = SqlEngine(useMoorExtensions: true); + final parseResult = engine.parseMoorFile(''); + + final suggestions = parseResult.autoCompleteEngine.suggestCompletions(0); + + expect(suggestions.anchor, 0); + expect(suggestions.suggestions, contains(hasCode('CREATE TABLE'))); + }); + + test('suggests completions for started expressions', () { + final engine = SqlEngine(useMoorExtensions: true); + final parseResult = engine.parseMoorFile('creat'); + + final suggestions = parseResult.autoCompleteEngine.suggestCompletions(0); + + expect(suggestions.anchor, 0); + expect(suggestions.suggestions, contains(hasCode('CREATE TABLE'))); + }); +} + +dynamic hasCode(code) => SuggestionWithCode(code); + +class SuggestionWithCode extends Matcher { + final Matcher codeMatcher; + + SuggestionWithCode(dynamic code) : codeMatcher = wrapMatcher(code); + + @override + Description describe(Description description) { + return description.add('suggests ').addDescriptionOf(codeMatcher); + } + + @override + bool matches(item, Map matchState) { + return item is Suggestion && codeMatcher.matches(item.code, matchState); + } +} From e807822f3f1083c55ee89147c6660869bba5affc Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 8 Sep 2019 10:29:59 +0200 Subject: [PATCH 062/117] Make plugin operate on the new MoorFile AST node --- .../lib/src/analyzer/moor/parser.dart | 2 +- moor_generator/lib/src/analyzer/results.dart | 7 ++-- .../lib/src/backends/plugin/plugin.dart | 20 +++++++++++- .../plugin/services/autocomplete.dart | 32 +++++++++++++++++++ .../src/backends/plugin/services/folding.dart | 4 +-- .../backends/plugin/services/highlights.dart | 6 ++-- .../src/backends/plugin/services/outline.dart | 4 +-- .../backends/plugin/services/requests.dart | 18 +++++++++++ sqlparser/lib/src/engine/sql_engine.dart | 6 ++-- 9 files changed, 81 insertions(+), 18 deletions(-) create mode 100644 moor_generator/lib/src/backends/plugin/services/autocomplete.dart diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index 316ba9f9..c36b0fe4 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -40,6 +40,6 @@ class MoorParser { final createdTables = createdReaders.map((r) => r.extractTable(task.mapper)).toList(); - return Future.value(ParsedMoorFile(createdTables)); + return Future.value(ParsedMoorFile(result, declaredTables: createdTables)); } } diff --git a/moor_generator/lib/src/analyzer/results.dart b/moor_generator/lib/src/analyzer/results.dart index 5e67e99a..7da2e026 100644 --- a/moor_generator/lib/src/analyzer/results.dart +++ b/moor_generator/lib/src/analyzer/results.dart @@ -22,10 +22,9 @@ class ParsedDartFile extends ParsedFile { } class ParsedMoorFile extends ParsedFile { - final List tokens; - final List statements; + final ParseResult parseResult; + MoorFile get parsedFile => parseResult.rootNode as MoorFile; final List declaredTables; - ParsedMoorFile(this.tokens, this.statements, - {this.declaredTables = const []}); + ParsedMoorFile(this.parseResult, {this.declaredTables = const []}); } diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart index 7145d3fb..dfc25502 100644 --- a/moor_generator/lib/src/backends/plugin/plugin.dart +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -1,15 +1,18 @@ import 'package:analyzer/src/context/context_root.dart'; // ignore: implementation_imports import 'package:analyzer/src/context/builder.dart'; // ignore: implementation_imports import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/plugin/completion_mixin.dart'; import 'package:analyzer_plugin/plugin/folding_mixin.dart'; import 'package:analyzer_plugin/plugin/highlights_mixin.dart'; import 'package:analyzer_plugin/plugin/outline_mixin.dart'; import 'package:analyzer_plugin/plugin/plugin.dart'; import 'package:analyzer_plugin/protocol/protocol_generated.dart' as plugin; +import 'package:analyzer_plugin/utilities/completion/completion_core.dart'; import 'package:analyzer_plugin/utilities/folding/folding.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; import 'package:analyzer_plugin/utilities/outline/outline.dart'; import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; +import 'package:moor_generator/src/backends/plugin/services/autocomplete.dart'; import 'package:moor_generator/src/backends/plugin/services/folding.dart'; import 'package:moor_generator/src/backends/plugin/services/highlights.dart'; import 'package:moor_generator/src/backends/plugin/services/outline.dart'; @@ -19,7 +22,7 @@ import 'backend/driver.dart'; import 'backend/logger.dart'; class MoorPlugin extends ServerPlugin - with OutlineMixin, HighlightsMixin, FoldingMixin { + with OutlineMixin, HighlightsMixin, FoldingMixin, CompletionMixin { MoorPlugin(ResourceProvider provider) : super(provider) { setupLogger(this); } @@ -105,4 +108,19 @@ class MoorPlugin extends ServerPlugin @override Future getFoldingRequest(String path) => _createMoorRequest(path); + + @override + List getCompletionContributors(String path) { + return [const MoorCompletingContributor()]; + } + + @override + Future getCompletionRequest( + plugin.CompletionGetSuggestionsParams parameters) async { + final path = parameters.file; + final driver = _moorDriverForPath(path); + final task = await driver.parseMoorFile(path); + + return MoorCompletionRequest(parameters.offset, resourceProvider, task); + } } diff --git a/moor_generator/lib/src/backends/plugin/services/autocomplete.dart b/moor_generator/lib/src/backends/plugin/services/autocomplete.dart new file mode 100644 index 00000000..4bfab9a9 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/autocomplete.dart @@ -0,0 +1,32 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:analyzer_plugin/utilities/completion/completion_core.dart'; +import 'package:moor_generator/src/backends/plugin/services/requests.dart'; + +class MoorCompletingContributor implements CompletionContributor { + const MoorCompletingContributor(); + + @override + Future computeSuggestions( + MoorCompletionRequest request, CompletionCollector collector) { + final autoComplete = request.task.lastResult.parseResult.autoCompleteEngine; + final results = autoComplete.suggestCompletions(request.offset); + + collector + ..offset = results.anchor + ..length = results.lengthBefore; + + for (var suggestion in results.suggestions) { + collector.addSuggestion(CompletionSuggestion( + CompletionSuggestionKind.KEYWORD, + suggestion.relevance, + suggestion.code, + -1, + -1, + false, + false, + )); + } + + return Future.value(); + } +} diff --git a/moor_generator/lib/src/backends/plugin/services/folding.dart b/moor_generator/lib/src/backends/plugin/services/folding.dart index 36f65b1d..9d88dcb2 100644 --- a/moor_generator/lib/src/backends/plugin/services/folding.dart +++ b/moor_generator/lib/src/backends/plugin/services/folding.dart @@ -11,9 +11,7 @@ class MoorFoldingContributor implements FoldingContributor { final moorRequest = request as MoorRequest; final visitor = _FoldingVisitor(collector); - for (var stmt in moorRequest.resolvedTask.lastResult.statements) { - stmt.accept(visitor); - } + moorRequest.resolvedTask.lastResult.parsedFile.accept(visitor); } } diff --git a/moor_generator/lib/src/backends/plugin/services/highlights.dart b/moor_generator/lib/src/backends/plugin/services/highlights.dart index 6f10528e..8efba5e3 100644 --- a/moor_generator/lib/src/backends/plugin/services/highlights.dart +++ b/moor_generator/lib/src/backends/plugin/services/highlights.dart @@ -18,11 +18,9 @@ class MoorHighlightContributor implements HighlightsContributor { final result = typedRequest.resolvedTask.lastResult; - for (var stmt in result.statements) { - stmt.accept(visitor); - } + result.parsedFile.accept(visitor); - for (var token in result.tokens) { + for (var token in result.parseResult.tokens) { if (token is KeywordToken) { final start = token.span.start.offset; final length = token.span.length; diff --git a/moor_generator/lib/src/backends/plugin/services/outline.dart b/moor_generator/lib/src/backends/plugin/services/outline.dart index 5335cab9..e10c6eb6 100644 --- a/moor_generator/lib/src/backends/plugin/services/outline.dart +++ b/moor_generator/lib/src/backends/plugin/services/outline.dart @@ -18,9 +18,7 @@ class MoorOutlineContributor implements OutlineContributor { libraryElement, 0, moorRequest.resolvedTask.content.length); final visitor = _OutlineVisitor(collector); - for (var stmt in moorRequest.resolvedTask.lastResult.statements) { - stmt.accept(visitor); - } + moorRequest.resolvedTask.lastResult.parsedFile.accept(visitor); collector.endElement(); } diff --git a/moor_generator/lib/src/backends/plugin/services/requests.dart b/moor_generator/lib/src/backends/plugin/services/requests.dart index bfd22ee4..767106d8 100644 --- a/moor_generator/lib/src/backends/plugin/services/requests.dart +++ b/moor_generator/lib/src/backends/plugin/services/requests.dart @@ -1,4 +1,5 @@ import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/utilities/completion/completion_core.dart'; import 'package:analyzer_plugin/utilities/folding/folding.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; import 'package:analyzer_plugin/utilities/outline/outline.dart'; @@ -14,3 +15,20 @@ class MoorRequest implements OutlineRequest, HighlightsRequest, FoldingRequest { @override String get path => resolvedTask.backendTask.entrypoint.toFilePath(); } + +// todo CompletionRequest likes not to be extended, but there is no suitable +// subclass. +class MoorCompletionRequest extends CompletionRequest { + @override + void checkAborted() {} + + @override + final int offset; + + @override + final ResourceProvider resourceProvider; + + final MoorTask task; + + MoorCompletionRequest(this.offset, this.resourceProvider, this.task); +} diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index e978e75c..1d257a7e 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -64,7 +64,8 @@ class SqlEngine { final moorFile = parser.moorFile(); - return ParseResult._(moorFile, tokens, parser.errors, content, autoComplete); + return ParseResult._( + moorFile, tokens, parser.errors, content, autoComplete); } /// Parses and analyzes the [sql] statement. The [AnalysisContext] returned @@ -130,5 +131,6 @@ class ParseResult { /// result. final AutoCompleteEngine autoCompleteEngine; - ParseResult._(this.rootNode, this.tokens, this.errors, this.sql, this.autoCompleteEngine); + ParseResult._(this.rootNode, this.tokens, this.errors, this.sql, + this.autoCompleteEngine); } From d79b04193c16a0db1204560439d510c13cd7f8ea Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 8 Sep 2019 11:22:36 +0200 Subject: [PATCH 063/117] Report syntax errors in moor files from the plugin --- .../src/backends/plugin/backend/driver.dart | 1 + .../backends/plugin/backend/file_tracker.dart | 13 +++++++ .../lib/src/backends/plugin/plugin.dart | 9 ++++- .../src/backends/plugin/services/errors.dart | 38 +++++++++++++++++++ 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 moor_generator/lib/src/backends/plugin/services/errors.dart diff --git a/moor_generator/lib/src/backends/plugin/backend/driver.dart b/moor_generator/lib/src/backends/plugin/backend/driver.dart index b141cf4a..bb16f51e 100644 --- a/moor_generator/lib/src/backends/plugin/backend/driver.dart +++ b/moor_generator/lib/src/backends/plugin/backend/driver.dart @@ -39,6 +39,7 @@ class MoorDriver implements AnalysisDriverGeneric { void dispose() { _scheduler.remove(this); dartDriver.dispose(); + _tracker.dispose(); } void handleFileChanged(String path) { diff --git a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart index 6e31ef5e..ef804f88 100644 --- a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart +++ b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart @@ -15,6 +15,13 @@ class FileTracker { final Map _trackedFiles = {}; final Set _currentPriority = {}; + final StreamController _computations = + StreamController.broadcast(); + + /// Streams that emits a [TrackedFile] when it has been worked on + /// successfully. + Stream get computations => _computations.stream; + FileTracker() { _pendingWork = PriorityQueue(_compareByPriority); } @@ -82,6 +89,8 @@ class FileTracker { final unit = _pendingWork.removeFirst(); worker(unit.path).then((result) { + _computations.add(unit); + for (var completer in unit._waiting) { completer.complete(result); } @@ -94,6 +103,10 @@ class FileTracker { }); } } + + void dispose() { + _computations.close(); + } } enum FileType { moor, unknown } diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart index dfc25502..fb151bd7 100644 --- a/moor_generator/lib/src/backends/plugin/plugin.dart +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -13,6 +13,7 @@ import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; import 'package:analyzer_plugin/utilities/outline/outline.dart'; import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; import 'package:moor_generator/src/backends/plugin/services/autocomplete.dart'; +import 'package:moor_generator/src/backends/plugin/services/errors.dart'; import 'package:moor_generator/src/backends/plugin/services/folding.dart'; import 'package:moor_generator/src/backends/plugin/services/highlights.dart'; import 'package:moor_generator/src/backends/plugin/services/outline.dart'; @@ -56,9 +57,15 @@ class MoorPlugin extends ServerPlugin // why exactly this is necessary final dartDriver = builder.buildDriver(analyzerRoot) ..results.listen((_) {}) // Consume the stream, otherwise we leak. - ..exceptions.listen((_) {}); // Consume the stream, otherwise we leak.; + ..exceptions.listen((_) {}); // Consume the stream, otherwise we leak. final tracker = FileTracker(); + final errorService = ErrorService(this); + + tracker.computations + .asyncMap((file) => tracker.results(file.path)) + .listen(errorService.handleMoorResult); + return MoorDriver(tracker, analysisDriverScheduler, dartDriver, fileContentOverlay, resourceProvider); } diff --git a/moor_generator/lib/src/backends/plugin/services/errors.dart b/moor_generator/lib/src/backends/plugin/services/errors.dart new file mode 100644 index 00000000..2e5aa067 --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/errors.dart @@ -0,0 +1,38 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:analyzer_plugin/protocol/protocol_generated.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/backends/plugin/plugin.dart'; + +const _parsingErrorCode = 'moor.parsingError'; + +/// Sends information about errors, lints and warnings encountered in a `.moor` +/// file to the analyzer. +class ErrorService { + final MoorPlugin plugin; + + ErrorService(this.plugin); + + void handleMoorResult(MoorTask completedTask) { + final result = completedTask.lastResult.parseResult; + final path = completedTask.backendTask.entrypoint.path; + + final errors = []; + + for (var error in result.errors) { + // this is a parsing error, high severity + final severity = AnalysisErrorSeverity.ERROR; + final type = AnalysisErrorType.SYNTACTIC_ERROR; + + final sourceSpan = error.token.span; + final start = sourceSpan.start; + final location = Location( + path, start.offset, sourceSpan.length, start.line, start.column); + + errors.add(AnalysisError( + severity, type, location, error.message, _parsingErrorCode)); + } + + final params = AnalysisErrorsParams(path, errors); + plugin.channel.sendNotification(params.toNotification()); + } +} From 1b7721a98f252e21ece3d444a46b151bdb2e5f3e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 8 Sep 2019 21:36:26 +0200 Subject: [PATCH 064/117] Plugin: Import assists for column nullability --- extras/plugin_example/README.md | 10 ++-- moor_generator/lib/src/analyzer/session.dart | 2 + .../lib/src/backends/plugin/plugin.dart | 28 ++++++++++- .../services/assists/assist_service.dart | 47 +++++++++++++++++++ .../services/assists/column_nullability.dart | 44 +++++++++++++++++ .../backends/plugin/services/requests.dart | 16 +++++++ sqlparser/lib/src/ast/ast.dart | 11 ++++- .../lib/src/ast/schema/column_definition.dart | 12 +++++ sqlparser/lib/src/engine/sql_engine.dart | 39 ++++++++++++++- sqlparser/lib/src/reader/parser/schema.dart | 8 +++- .../engine/find_node_by_position_test.dart | 19 ++++++++ 11 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart create mode 100644 moor_generator/lib/src/backends/plugin/services/assists/column_nullability.dart create mode 100644 sqlparser/test/engine/find_node_by_position_test.dart diff --git a/extras/plugin_example/README.md b/extras/plugin_example/README.md index d7d12cd4..8b3bed47 100644 --- a/extras/plugin_example/README.md +++ b/extras/plugin_example/README.md @@ -6,7 +6,7 @@ plugin. To set up the plugin, run the following steps 1. Change the file `moor/tools/analyzer_plugin/pubspec.yaml` so that the `dependency_overrides` section points to the location where you cloned this repository. This is needed because we - can't use relative paths for dependencies in analyzer plugins yet. + can't use relative paths for dependencies in analyzer plugins yet- see https://dartbug.com/35281 2. In VS Code, change `dart.additionalAnalyzerFileExtensions` to include `moor` files: ```json { @@ -15,10 +15,14 @@ plugin. To set up the plugin, run the following steps ] } ``` + To diagnose errors with the plugin, turning on the diagnostics server by setting a + `dart.analyzerDiagnosticsPort` and enabling the instrumentation log via `dart.analyzerInstrumentationLogFile` + is recommended as well. 3. If you already had the project open, close and re-open VS Code. Otherwise, simply open this project. -4. Type around in a `.moor` file - notice how you still don't get syntax highlighting because - VS Code required a static grammar for files and can't use a language server for that :( +4. Type around in a `.moor` file. +5. Notice how you don't see anything (https://github.com/Dart-Code/Dart-Code/issues/1981), but + at least the plugin output appears in the instrumentation log. Debugging plugins is not fun. See the [docs](https://github.com/dart-lang/sdk/blob/master/pkg/analyzer_plugin/doc/tutorial/debugging.md) on some general guidance, and good luck. Enabling the analyzer diagnostics server can help. diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 9aedfd14..d9f4495c 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -42,6 +42,8 @@ abstract class FileTask { final ErrorSink errors = ErrorSink(); + String get path => backendTask.entrypoint.path; + FileTask(this.backendTask, this.session); void reportError(MoorError error) => errors.report(error); diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart index fb151bd7..73c29c28 100644 --- a/moor_generator/lib/src/backends/plugin/plugin.dart +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -1,17 +1,20 @@ import 'package:analyzer/src/context/context_root.dart'; // ignore: implementation_imports import 'package:analyzer/src/context/builder.dart'; // ignore: implementation_imports import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/plugin/assist_mixin.dart'; import 'package:analyzer_plugin/plugin/completion_mixin.dart'; import 'package:analyzer_plugin/plugin/folding_mixin.dart'; import 'package:analyzer_plugin/plugin/highlights_mixin.dart'; import 'package:analyzer_plugin/plugin/outline_mixin.dart'; import 'package:analyzer_plugin/plugin/plugin.dart'; import 'package:analyzer_plugin/protocol/protocol_generated.dart' as plugin; +import 'package:analyzer_plugin/utilities/assist/assist.dart'; import 'package:analyzer_plugin/utilities/completion/completion_core.dart'; import 'package:analyzer_plugin/utilities/folding/folding.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; import 'package:analyzer_plugin/utilities/outline/outline.dart'; import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; +import 'package:moor_generator/src/backends/plugin/services/assists/assist_service.dart'; import 'package:moor_generator/src/backends/plugin/services/autocomplete.dart'; import 'package:moor_generator/src/backends/plugin/services/errors.dart'; import 'package:moor_generator/src/backends/plugin/services/folding.dart'; @@ -23,7 +26,12 @@ import 'backend/driver.dart'; import 'backend/logger.dart'; class MoorPlugin extends ServerPlugin - with OutlineMixin, HighlightsMixin, FoldingMixin, CompletionMixin { + with + OutlineMixin, + HighlightsMixin, + FoldingMixin, + CompletionMixin, + AssistsMixin { MoorPlugin(ResourceProvider provider) : super(provider) { setupLogger(this); } @@ -118,7 +126,7 @@ class MoorPlugin extends ServerPlugin @override List getCompletionContributors(String path) { - return [const MoorCompletingContributor()]; + return const [MoorCompletingContributor()]; } @override @@ -130,4 +138,20 @@ class MoorPlugin extends ServerPlugin return MoorCompletionRequest(parameters.offset, resourceProvider, task); } + + @override + List getAssistContributors(String path) { + return const [AssistService()]; + } + + @override + Future getAssistRequest( + plugin.EditGetAssistsParams parameters) async { + final path = parameters.file; + final driver = _moorDriverForPath(path); + final task = await driver.parseMoorFile(path); + + return MoorAssistRequest( + task, parameters.length, parameters.offset, resourceProvider); + } } diff --git a/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart b/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart new file mode 100644 index 00000000..bcd215fa --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart @@ -0,0 +1,47 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:analyzer_plugin/protocol/protocol_generated.dart'; +import 'package:analyzer_plugin/utilities/assist/assist.dart'; +import 'package:moor_generator/src/backends/plugin/services/requests.dart'; +import 'package:sqlparser/sqlparser.dart'; + +part 'column_nullability.dart'; + +class AssistService implements AssistContributor { + const AssistService(); + + final _nullability = const ColumnNullability(); + + @override + void computeAssists(AssistRequest request, AssistCollector collector) { + final moorRequest = request as MoorAssistRequest; + final parseResult = moorRequest.task.lastResult.parseResult; + final relevantNodes = + parseResult.findNodesAtPosition(request.offset, length: request.length); + + for (var node in relevantNodes.expand((node) => node.selfAndParents)) { + _handleNode(collector, node, moorRequest.task.path); + } + } + + void _handleNode(AssistCollector collector, AstNode node, String path) { + if (node is ColumnDefinition) { + _nullability.contribute(collector, node, path); + } + } +} + +abstract class _AssistOnNodeContributor { + const _AssistOnNodeContributor(); + + void contribute(AssistCollector collector, T node, String path); +} + +class AssistId { + final String id; + final int priority; + + const AssistId._(this.id, this.priority); + + static const makeNullable = AssistId._('make_column_nullable', 100); + static const makeNotNull = AssistId._('make_column_not_nullable', 10); +} diff --git a/moor_generator/lib/src/backends/plugin/services/assists/column_nullability.dart b/moor_generator/lib/src/backends/plugin/services/assists/column_nullability.dart new file mode 100644 index 00000000..bad5ed6a --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/services/assists/column_nullability.dart @@ -0,0 +1,44 @@ +part of 'assist_service.dart'; + +class ColumnNullability extends _AssistOnNodeContributor { + const ColumnNullability(); + + @override + void contribute( + AssistCollector collector, ColumnDefinition node, String path) { + final notNull = node.findConstraint(); + + if (notNull == null) { + // there is no not-null constraint on this column, suggest to add one at + // the end of the definition + final end = node.lastPosition; + final id = AssistId.makeNotNull; + + collector.addAssist(PrioritizedSourceChange( + id.priority, + SourceChange('Add a NOT NULL constraint', id: id.id, edits: [ + SourceFileEdit( + path, + -1, + edits: [ + SourceEdit(end, 0, ' NOT NULL'), + ], + ) + ]), + )); + } else { + // suggest to remove the NOT NULL constraint, e.g. to make this column + // nullable + final id = AssistId.makeNullable; + + collector.addAssist(PrioritizedSourceChange( + id.priority, + SourceChange('Make this column nullable', id: id.id, edits: [ + SourceFileEdit(path, -1, edits: [ + SourceEdit(notNull.firstPosition, notNull.lastPosition, '') + ]) + ]), + )); + } + } +} diff --git a/moor_generator/lib/src/backends/plugin/services/requests.dart b/moor_generator/lib/src/backends/plugin/services/requests.dart index 767106d8..9a9d2c5b 100644 --- a/moor_generator/lib/src/backends/plugin/services/requests.dart +++ b/moor_generator/lib/src/backends/plugin/services/requests.dart @@ -1,4 +1,5 @@ import 'package:analyzer/file_system/file_system.dart'; +import 'package:analyzer_plugin/utilities/assist/assist.dart'; import 'package:analyzer_plugin/utilities/completion/completion_core.dart'; import 'package:analyzer_plugin/utilities/folding/folding.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; @@ -32,3 +33,18 @@ class MoorCompletionRequest extends CompletionRequest { MoorCompletionRequest(this.offset, this.resourceProvider, this.task); } + +class MoorAssistRequest extends AssistRequest { + final MoorTask task; + + @override + final int length; + + @override + final int offset; + + @override + final ResourceProvider resourceProvider; + + MoorAssistRequest(this.task, this.length, this.offset, this.resourceProvider); +} diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index 2734145b..003f41ac 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -56,7 +56,10 @@ abstract class AstNode { /// The last position that belongs to node, exclusive. Not set for all nodes. int get lastPosition => last.span.end.offset; - FileSpan get span => first.span.expand(last.span); + FileSpan get span { + if (first == null || last == null) return null; + return first.span.expand(last.span); + } /// Sets the [AstNode.first] and [AstNode.last] property in one go. void setSpan(Token first, Token last) { @@ -74,6 +77,12 @@ abstract class AstNode { } } + /// Returns an iterable containing `this` node and all [parents]. + Iterable get selfAndParents sync* { + yield this; + yield* parents; + } + /// Recursively returns all descendants of this node, e.g. its children, their /// children and so on. The tree will be pre-order traversed. Iterable get allDescendants sync* { diff --git a/sqlparser/lib/src/ast/schema/column_definition.dart b/sqlparser/lib/src/ast/schema/column_definition.dart index d875e5fb..61bd3d67 100644 --- a/sqlparser/lib/src/ast/schema/column_definition.dart +++ b/sqlparser/lib/src/ast/schema/column_definition.dart @@ -21,6 +21,15 @@ class ColumnDefinition extends AstNode { bool contentEquals(ColumnDefinition other) { return other.columnName == columnName && other.typeName == typeName; } + + /// Finds a constraint of type [T], or null, if none is set. + T findConstraint() { + final typedConstraints = constraints.whereType().iterator; + if (typedConstraints.moveNext()) { + return typedConstraints.current; + } + return null; + } } /// https://www.sqlite.org/syntax/column-constraint.html @@ -77,6 +86,9 @@ enum ConflictClause { rollback, abort, fail, ignore, replace } class NotNull extends ColumnConstraint { final ConflictClause onConflict; + Token not; + Token $null; + NotNull(String name, {this.onConflict}) : super(name); @override diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 1d257a7e..139f7f4f 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -1,3 +1,5 @@ +import 'dart:collection'; + import 'package:sqlparser/src/analysis/analysis.dart'; import 'package:sqlparser/src/ast/ast.dart'; import 'package:sqlparser/src/engine/autocomplete/engine.dart'; @@ -87,7 +89,6 @@ class SqlEngine { /// [registerTable] before calling this method. AnalysisContext analyzeParsed(ParseResult result) { final node = result.rootNode; - const SetParentVisitor().startAtRoot(node); final context = AnalysisContext(node, result.sql); final scope = _constructRootScope(); @@ -132,5 +133,39 @@ class ParseResult { final AutoCompleteEngine autoCompleteEngine; ParseResult._(this.rootNode, this.tokens, this.errors, this.sql, - this.autoCompleteEngine); + this.autoCompleteEngine) { + const SetParentVisitor().startAtRoot(rootNode); + } + + /// Attempts to find the most relevant (bottom-most in the AST) nodes that + /// intersects with the source range from [offset] to [offset] + [length]. + List findNodesAtPosition(int offset, {int length = 0}) { + if (tokens.isEmpty || rootNode == null) return const []; + + final candidates = {}; + final unchecked = Queue(); + unchecked.add(rootNode); + + while (unchecked.isNotEmpty) { + final node = unchecked.removeFirst(); + + final span = node.span; + final start = span.start.offset; + final end = span.end.offset; + + final hasIntersection = !(end < offset || start > offset + length); + if (hasIntersection) { + // this node matches. As we want to find the bottom-most node in the AST + // that matches, this means that the parent is no longer a candidate. + candidates.add(node); + candidates.remove(node.parent); + + // assume that the span of a node is a superset of the span of any + // child, so each child could potentially be interesting. + unchecked.addAll(node.childNodes); + } + } + + return candidates.toList(); + } } diff --git a/sqlparser/lib/src/reader/parser/schema.dart b/sqlparser/lib/src/reader/parser/schema.dart index e0fe8cbc..a0aa9363 100644 --- a/sqlparser/lib/src/reader/parser/schema.dart +++ b/sqlparser/lib/src/reader/parser/schema.dart @@ -129,10 +129,14 @@ mixin SchemaParser on ParserBase { ..setSpan(first, _previous); } if (_matchOne(TokenType.not)) { - _consume(TokenType.$null, 'Expected NULL to complete NOT NULL'); + final notToken = _previous; + final nullToken = + _consume(TokenType.$null, 'Expected NULL to complete NOT NULL'); return NotNull(resolvedName, onConflict: _conflictClauseOrNull()) - ..setSpan(first, _previous); + ..setSpan(first, _previous) + ..not = notToken + ..$null = nullToken; } if (_matchOne(TokenType.unique)) { return UniqueColumn(resolvedName, _conflictClauseOrNull()) diff --git a/sqlparser/test/engine/find_node_by_position_test.dart b/sqlparser/test/engine/find_node_by_position_test.dart new file mode 100644 index 00000000..31d3d8bb --- /dev/null +++ b/sqlparser/test/engine/find_node_by_position_test.dart @@ -0,0 +1,19 @@ +import 'package:sqlparser/sqlparser.dart'; +import 'package:test/test.dart'; + +void main() { + test('finds the most relevant node', () { + final engine = SqlEngine(); + final result = engine.parse('SELECT * FROM tbl;'); + // | this is offset 8 + // | this is offset 17 + + final mostRelevantAtStar = result.findNodesAtPosition(8); + expect(mostRelevantAtStar.length, 1); + expect(mostRelevantAtStar.single, const TypeMatcher()); + + final mostRelevantAtTbl = result.findNodesAtPosition(17, length: 2); + expect(mostRelevantAtTbl.length, 1); + expect(mostRelevantAtTbl.single, const TypeMatcher()); + }); +} From a5cecd3ba665069539dbd8fb5ec9506321c13214 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 9 Sep 2019 20:34:14 +0200 Subject: [PATCH 065/117] Support moor file AST in the plugin --- .../src/backends/plugin/services/folding.dart | 14 ++++++++++++++ .../src/backends/plugin/services/highlights.dart | 2 +- .../src/backends/plugin/services/outline.dart | 16 ++++++++-------- sqlparser/lib/src/ast/moor/moor_file.dart | 8 ++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/moor_generator/lib/src/backends/plugin/services/folding.dart b/moor_generator/lib/src/backends/plugin/services/folding.dart index 9d88dcb2..f065f50b 100644 --- a/moor_generator/lib/src/backends/plugin/services/folding.dart +++ b/moor_generator/lib/src/backends/plugin/services/folding.dart @@ -20,6 +20,20 @@ class _FoldingVisitor extends RecursiveVisitor { _FoldingVisitor(this.collector); + @override + void visitMoorFile(MoorFile e) { + // construct a folding region for import statements + final imports = e.imports.toList(); + if (imports.isNotEmpty) { + final first = imports.first.firstPosition; + final last = imports.last.lastPosition; + + collector.addRegion(first, last - first, FoldingKind.DIRECTIVES); + } + + super.visitChildren(e); + } + @override void visitCreateTableStatement(CreateTableStatement e) { final startBody = e.openingBracket; diff --git a/moor_generator/lib/src/backends/plugin/services/highlights.dart b/moor_generator/lib/src/backends/plugin/services/highlights.dart index 8efba5e3..a3b8c690 100644 --- a/moor_generator/lib/src/backends/plugin/services/highlights.dart +++ b/moor_generator/lib/src/backends/plugin/services/highlights.dart @@ -21,7 +21,7 @@ class MoorHighlightContributor implements HighlightsContributor { result.parsedFile.accept(visitor); for (var token in result.parseResult.tokens) { - if (token is KeywordToken) { + if (token is KeywordToken && !token.isIdentifier) { final start = token.span.start.offset; final length = token.span.length; collector.addRegion(start, length, HighlightRegionType.BUILT_IN); diff --git a/moor_generator/lib/src/backends/plugin/services/outline.dart b/moor_generator/lib/src/backends/plugin/services/outline.dart index e10c6eb6..328cf5ea 100644 --- a/moor_generator/lib/src/backends/plugin/services/outline.dart +++ b/moor_generator/lib/src/backends/plugin/services/outline.dart @@ -11,16 +11,9 @@ class MoorOutlineContributor implements OutlineContributor { @override void computeOutline(OutlineRequest request, OutlineCollector collector) { final moorRequest = request as MoorRequest; - final file = moorRequest.path; - - final libraryElement = Element(ElementKind.FILE, file, _defaultFlags); - collector.startElement( - libraryElement, 0, moorRequest.resolvedTask.content.length); final visitor = _OutlineVisitor(collector); moorRequest.resolvedTask.lastResult.parsedFile.accept(visitor); - - collector.endElement(); } } @@ -48,7 +41,14 @@ class _OutlineVisitor extends RecursiveVisitor { @override void visitColumnDefinition(ColumnDefinition e) { - _startElement(ElementKind.FIELD, e.columnName, e); + _startElement(ElementKind.FIELD, e.columnName, e)..returnType = e.typeName; + super.visitChildren(e); + collector.endElement(); + } + + @override + void visitMoorDeclaredStatement(DeclaredStatement e) { + _startElement(ElementKind.TOP_LEVEL_VARIABLE, e.name, e); super.visitChildren(e); collector.endElement(); } diff --git a/sqlparser/lib/src/ast/moor/moor_file.dart b/sqlparser/lib/src/ast/moor/moor_file.dart index d451d14f..30005bef 100644 --- a/sqlparser/lib/src/ast/moor/moor_file.dart +++ b/sqlparser/lib/src/ast/moor/moor_file.dart @@ -3,6 +3,10 @@ part of '../ast.dart'; /// Something that can appear as a top-level declaration inside a `.moor` file. abstract class PartOfMoorFile implements Statement {} +/// A moor file. +/// +/// A moor file consists of [ImportStatement], followed by ddl statements, +/// followed by [DeclaredStatement]s. class MoorFile extends AstNode { final List statements; @@ -14,6 +18,10 @@ class MoorFile extends AstNode { @override Iterable get childNodes => statements; + /// Returns the imports defined in this file. + Iterable get imports => + childNodes.whereType(); + @override bool contentEquals(MoorFile other) => true; } From 28f13abfc350701d6f63356253946254e8c16067 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 11 Sep 2019 21:30:42 +0200 Subject: [PATCH 066/117] Fix type converters loosing generic information Fixes #144 --- moor_generator/lib/src/model/specified_column.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moor_generator/lib/src/model/specified_column.dart b/moor_generator/lib/src/model/specified_column.dart index 6e63d6fc..0ae6477d 100644 --- a/moor_generator/lib/src/model/specified_column.dart +++ b/moor_generator/lib/src/model/specified_column.dart @@ -110,7 +110,7 @@ class SpecifiedColumn { /// table has declared an `IntColumn`, the matching dart type name would be [int]. String get dartTypeName { if (typeConverter != null) { - return typeConverter.mappedType?.name; + return typeConverter.mappedType?.displayName; } return variableTypeName; } From e394292978fdf7ebabc6096cad4060458848d813 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 11 Sep 2019 23:00:46 +0200 Subject: [PATCH 067/117] Refactor the backend (again) We now have a two-pass parsing and analyzing model that supports (zyklic) imports between Dart and moor files. --- .../lib/src/analyzer/dart/column_parser.dart | 8 +- .../lib/src/analyzer/dart/parser.dart | 11 +- .../lib/src/analyzer/dart/table_parser.dart | 6 +- .../lib/src/analyzer/dart/use_dao_parser.dart | 40 +++- .../src/analyzer/dart/use_moor_parser.dart | 17 +- .../analyzer/moor/inline_dart_resolver.dart | 8 +- .../lib/src/analyzer/moor/parser.dart | 18 +- .../lib/src/analyzer/runner/file_graph.dart | 115 +++++++++++ .../lib/src/analyzer/{ => runner}/inputs.dart | 0 .../src/analyzer/{ => runner}/results.dart | 12 +- .../lib/src/analyzer/runner/steps.dart | 174 ++++++++++++++++ .../lib/src/analyzer/runner/task.dart | 151 ++++++++++++++ moor_generator/lib/src/analyzer/session.dart | 185 +++--------------- .../src/analyzer/sql_queries/sql_parser.dart | 31 ++- moor_generator/lib/src/backends/backend.dart | 21 +- .../lib/src/backends/build/build_backend.dart | 26 ++- .../build/generators/dao_generator.dart | 80 +++----- .../build/generators/moor_generator.dart | 39 ++-- .../lib/src/backends/build/moor_builder.dart | 28 ++- .../lib/src/model/specified_dao.dart | 12 -- .../lib/src/model/specified_database.dart | 13 -- .../lib/src/model/specified_db_classes.dart | 40 ++++ moor_generator/lib/src/model/sql_query.dart | 7 + .../lib/src/writer/database_writer.dart | 4 +- 24 files changed, 706 insertions(+), 340 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/runner/file_graph.dart rename moor_generator/lib/src/analyzer/{ => runner}/inputs.dart (100%) rename moor_generator/lib/src/analyzer/{ => runner}/results.dart (70%) create mode 100644 moor_generator/lib/src/analyzer/runner/steps.dart create mode 100644 moor_generator/lib/src/analyzer/runner/task.dart delete mode 100644 moor_generator/lib/src/model/specified_dao.dart delete mode 100644 moor_generator/lib/src/model/specified_database.dart create mode 100644 moor_generator/lib/src/model/specified_db_classes.dart diff --git a/moor_generator/lib/src/analyzer/dart/column_parser.dart b/moor_generator/lib/src/analyzer/dart/column_parser.dart index 97ef1e9e..9964daf9 100644 --- a/moor_generator/lib/src/analyzer/dart/column_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/column_parser.dart @@ -49,7 +49,7 @@ class ColumnParser { final expr = base.returnExpressionOfMethod(getter); if (!(expr is FunctionExpressionInvocation)) { - base.task.reportError(ErrorInDartCode( + base.step.reportError(ErrorInDartCode( affectedElement: getter.declaredElement, message: _errorMessage, severity: Severity.criticalError, @@ -81,7 +81,7 @@ class ColumnParser { switch (methodName) { case _methodNamed: if (foundExplicitName != null) { - base.task.reportError( + base.step.reportError( ErrorInDartCode( severity: Severity.warning, affectedElement: getter.declaredElement, @@ -94,7 +94,7 @@ class ColumnParser { foundExplicitName = base.readStringLiteral( remainingExpr.argumentList.arguments.first, () { - base.task.reportError( + base.step.reportError( ErrorInDartCode( severity: Severity.error, affectedElement: getter.declaredElement, @@ -128,7 +128,7 @@ class ColumnParser { case _methodCustomConstraint: foundCustomConstraint = base.readStringLiteral( remainingExpr.argumentList.arguments.first, () { - base.task.reportError( + base.step.reportError( ErrorInDartCode( severity: Severity.warning, affectedElement: getter.declaredElement, diff --git a/moor_generator/lib/src/analyzer/dart/parser.dart b/moor_generator/lib/src/analyzer/dart/parser.dart index c6aa77e6..f61c3942 100644 --- a/moor_generator/lib/src/analyzer/dart/parser.dart +++ b/moor_generator/lib/src/analyzer/dart/parser.dart @@ -5,10 +5,9 @@ import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:moor/sqlite_keywords.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/model/specified_column.dart'; -import 'package:moor_generator/src/model/specified_dao.dart'; -import 'package:moor_generator/src/model/specified_database.dart'; +import 'package:moor_generator/src/model/specified_db_classes.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/used_type_converter.dart'; import 'package:moor_generator/src/utils/names.dart'; @@ -22,12 +21,12 @@ part 'use_dao_parser.dart'; part 'use_moor_parser.dart'; class MoorDartParser { - final DartTask task; + final ParseDartStep step; ColumnParser _columnParser; TableParser _tableParser; - MoorDartParser(this.task) { + MoorDartParser(this.step) { _columnParser = ColumnParser(this); _tableParser = TableParser(this); } @@ -46,7 +45,7 @@ class MoorDartParser { final body = method.body; if (!(body is ExpressionFunctionBody)) { - task.reportError(ErrorInDartCode( + step.reportError(ErrorInDartCode( affectedElement: method.declaredElement, severity: Severity.criticalError, message: diff --git a/moor_generator/lib/src/analyzer/dart/table_parser.dart b/moor_generator/lib/src/analyzer/dart/table_parser.dart index 6f214a27..6cd96b31 100644 --- a/moor_generator/lib/src/analyzer/dart/table_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/table_parser.dart @@ -60,7 +60,7 @@ class TableParser { tableNameDeclaration.node as MethodDeclaration); final tableName = base.readStringLiteral(returnExpr, () { - base.task.reportError(ErrorInDartCode( + base.step.reportError(ErrorInDartCode( severity: Severity.criticalError, message: 'This getter must return a string literal, and do nothing more', @@ -81,7 +81,7 @@ class TableParser { final ast = resolved.node as MethodDeclaration; final body = ast.body; if (body is! ExpressionFunctionBody) { - base.task.reportError(ErrorInDartCode( + base.step.reportError(ErrorInDartCode( affectedElement: primaryKeyGetter, message: 'This must return a set literal using the => syntax!')); return null; @@ -100,7 +100,7 @@ class TableParser { } } } else { - base.task.reportError(ErrorInDartCode( + base.step.reportError(ErrorInDartCode( affectedElement: primaryKeyGetter, message: 'This must return a set literal!')); } diff --git a/moor_generator/lib/src/analyzer/dart/use_dao_parser.dart b/moor_generator/lib/src/analyzer/dart/use_dao_parser.dart index 796cee79..61e28d55 100644 --- a/moor_generator/lib/src/analyzer/dart/use_dao_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/use_dao_parser.dart @@ -1,14 +1,36 @@ part of 'parser.dart'; class UseDaoParser { - final DartTask dartTask; + final ParseDartStep step; - UseDaoParser(this.dartTask); + UseDaoParser(this.step); /// If [element] has a `@UseDao` annotation, parses the database model /// declared by that class and the referenced tables. Future parseDao( ClassElement element, ConstantReader annotation) async { + final dbType = element.supertype; + if (dbType.name != 'DatabaseAccessor') { + step.reportError(ErrorInDartCode( + affectedElement: element, + severity: Severity.criticalError, + message: 'This class must directly inherit from DatabaseAccessor', + )); + return null; + } + + // inherits from DatabaseAccessor, we want to know which T + final dbImpl = dbType.typeArguments.single; + if (dbImpl.isDynamic) { + step.reportError(ErrorInDartCode( + affectedElement: element, + severity: Severity.criticalError, + message: 'This class must inherit from DatabaseAccessor, where T ' + 'is an actual type of a database.', + )); + return null; + } + final tableTypes = annotation.peek('tables')?.listValue?.map((obj) => obj.toTypeValue()) ?? []; @@ -18,15 +40,13 @@ class UseDaoParser { .read('include') .objectValue .toSetValue() - ?.map((e) => e.toStringValue()) ?? - {}; + ?.map((e) => e.toStringValue()) + ?.toList() ?? + []; - final parsedTables = await dartTask.parseTables(tableTypes, element); - parsedTables.addAll(await dartTask.resolveIncludes(includes)); + final parsedTables = await step.parseTables(tableTypes, element); + final parsedQueries = step.readDeclaredQueries(queryStrings); - final parsedQueries = - await dartTask.parseQueries(queryStrings, parsedTables); - - return SpecifiedDao(element, parsedTables, parsedQueries); + return SpecifiedDao(element, dbImpl, parsedTables, includes, parsedQueries); } } diff --git a/moor_generator/lib/src/analyzer/dart/use_moor_parser.dart b/moor_generator/lib/src/analyzer/dart/use_moor_parser.dart index 80ec18f3..2d0b3a43 100644 --- a/moor_generator/lib/src/analyzer/dart/use_moor_parser.dart +++ b/moor_generator/lib/src/analyzer/dart/use_moor_parser.dart @@ -1,9 +1,9 @@ part of 'parser.dart'; class UseMoorParser { - final DartTask task; + final ParseDartStep step; - UseMoorParser(this.task); + UseMoorParser(this.step); /// If [element] has a `@UseMoor` annotation, parses the database model /// declared by that class and the referenced tables. @@ -18,16 +18,17 @@ class UseMoorParser { .read('include') .objectValue .toSetValue() - ?.map((e) => e.toStringValue()) ?? - {}; + ?.map((e) => e.toStringValue()) + ?.toList() ?? + []; - final parsedTables = await task.parseTables(tableTypes, element); - parsedTables.addAll(await task.resolveIncludes(includes)); + final parsedTables = await step.parseTables(tableTypes, element); - final parsedQueries = await task.parseQueries(queryStrings, parsedTables); + final parsedQueries = step.readDeclaredQueries(queryStrings); final daoTypes = _readDaoTypes(annotation); - return SpecifiedDatabase(element, parsedTables, daoTypes, parsedQueries); + return SpecifiedDatabase( + element, parsedTables, daoTypes, includes, parsedQueries); } List _readDaoTypes(ConstantReader annotation) { diff --git a/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart b/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart index aee7b0a8..e5774e2b 100644 --- a/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart +++ b/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart @@ -1,6 +1,6 @@ import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/type.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; /// Resolves the type of Dart expressions given as a string. The /// [importStatements] are used to discover types. @@ -18,13 +18,13 @@ import 'package:moor_generator/src/analyzer/session.dart'; /// of the top-level `expr` variable in that source. class InlineDartResolver { final List importStatements = []; - final MoorTask task; + final ParseMoorFile step; - InlineDartResolver(this.task); + InlineDartResolver(this.step); Future resolveDartTypeOf(String expression) async { final template = _createDartTemplate(expression); - final unit = await task.backendTask.parseSource(template); + final unit = await step.task.backend.parseSource(template); final declaration = unit.declarations.single as TopLevelVariableDeclaration; return declaration.variables.variables.single.initializer.staticType; diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index c36b0fe4..66e214ab 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -1,17 +1,17 @@ import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/analyzer/moor/create_table_reader.dart'; -import 'package:moor_generator/src/analyzer/results.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; import 'package:sqlparser/sqlparser.dart'; class MoorParser { - final MoorTask task; + final ParseMoorFile step; - MoorParser(this.task); + MoorParser(this.step); Future parseAndAnalyze() { final result = - SqlEngine(useMoorExtensions: true).parseMoorFile(task.content); + SqlEngine(useMoorExtensions: true).parseMoorFile(step.content); final parsedFile = result.rootNode as MoorFile; final createdReaders = []; @@ -19,11 +19,11 @@ class MoorParser { for (var parsedStmt in parsedFile.statements) { if (parsedStmt is ImportStatement) { final importStmt = parsedStmt; - task.inlineDartResolver.importStatements.add(importStmt.importedFile); + step.inlineDartResolver.importStatements.add(importStmt.importedFile); } else if (parsedStmt is CreateTableStatement) { createdReaders.add(CreateTableReader(parsedStmt)); } else { - task.reportError(ErrorInMoorFile( + step.reportError(ErrorInMoorFile( span: parsedStmt.span, message: 'At the moment, only CREATE TABLE statements are supported' 'in .moor files')); @@ -31,14 +31,14 @@ class MoorParser { } for (var error in result.errors) { - task.reportError(ErrorInMoorFile( + step.reportError(ErrorInMoorFile( span: error.token.span, message: error.message, )); } final createdTables = - createdReaders.map((r) => r.extractTable(task.mapper)).toList(); + createdReaders.map((r) => r.extractTable(step.mapper)).toList(); return Future.value(ParsedMoorFile(result, declaredTables: createdTables)); } diff --git a/moor_generator/lib/src/analyzer/runner/file_graph.dart b/moor_generator/lib/src/analyzer/runner/file_graph.dart new file mode 100644 index 00000000..db722e05 --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/file_graph.dart @@ -0,0 +1,115 @@ +import 'dart:collection'; + +import 'package:collection/collection.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; + +import '../errors.dart'; + +/// Represents found files as nodes and import statements as edges. +class FileGraph { + final Map _files; + final UnmodifiableListView files; + + final Map> _imports = {}; + final Map> _transposedImports = {}; + + // using a factory constructor here to the readonly fields can be final + factory FileGraph() { + final files = {}; + final filesView = UnmodifiableListView(files.values); + + return FileGraph._(files, filesView); + } + + FileGraph._(this._files, this.files); + + /// Checks if a file with the given [uri] is registered in this graph. If it's + /// not, it will be created via [create] and inserted. + FoundFile registerFile(Uri uri, FoundFile Function() create) { + return _files.putIfAbsent(uri, create); + } + + /// Finds all files that [file] transitively imports (or, if [transposed] is + /// true, is transitively imported by). + Iterable crawl(FoundFile file, {bool transposed = false}) { + final edges = transposed ? _transposedImports : _imports; + + // breadth first search + final found = {}; + final unhandled = Queue()..add(file); + + while (unhandled.isNotEmpty) { + final file = unhandled.removeFirst(); + final neighbors = edges[file]; + + if (neighbors != null) { + for (var neighbor in neighbors) { + // if the neighbor wasn't in the set, also add to unhandled nodes so + // that we crawl its imports later. + if (found.add(neighbor)) { + unhandled.add(neighbor); + } + } + } + } + + return found; + } + + void setImports(FoundFile file, List updatedImports) { + registerFile(file.uri, () => file); + + // clear old imports, we also need to take the transposed imports into + // account here + if (_imports.containsKey(file)) { + for (var oldImport in _imports[file]) { + _transposedImports[oldImport]?.remove(file); + } + _imports.remove(file); + } + + _imports[file] = updatedImports; + + for (var newImport in updatedImports) { + _transposedImports.putIfAbsent(newImport, () => []).add(file); + } + } +} + +enum FileType { moor, dart, other } + +enum FileState { + /// The file was discovered, but not handled yet + dirty, + + /// The file completed the first step in the analysis task, which means that + /// the overall structure was parsed. + parsed, + + /// The file is fully analyzed, which means that all information is fully + /// available + analyzed +} + +class FoundFile { + /// The uri of this file, which can be an asset on the build backend or a + /// `file://` uri on the analysis plugin backend. + final Uri uri; + final FileType type; + + FileResult currentResult; + FileState state; + ErrorSink errors; + + String get shortName => uri.pathSegments.last; + + FoundFile(this.uri, this.type); + + @override + int get hashCode => uri.hashCode; + + @override + bool operator ==(other) { + return identical(this, other) || other is FoundFile && other.uri == uri; + } +} diff --git a/moor_generator/lib/src/analyzer/inputs.dart b/moor_generator/lib/src/analyzer/runner/inputs.dart similarity index 100% rename from moor_generator/lib/src/analyzer/inputs.dart rename to moor_generator/lib/src/analyzer/runner/inputs.dart diff --git a/moor_generator/lib/src/analyzer/results.dart b/moor_generator/lib/src/analyzer/runner/results.dart similarity index 70% rename from moor_generator/lib/src/analyzer/results.dart rename to moor_generator/lib/src/analyzer/runner/results.dart index 7da2e026..cb3adb37 100644 --- a/moor_generator/lib/src/analyzer/results.dart +++ b/moor_generator/lib/src/analyzer/runner/results.dart @@ -1,19 +1,21 @@ import 'package:meta/meta.dart'; import 'package:analyzer/dart/element/element.dart'; -import 'package:moor_generator/src/model/specified_dao.dart'; -import 'package:moor_generator/src/model/specified_database.dart'; +import 'package:moor_generator/src/model/specified_db_classes.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:sqlparser/sqlparser.dart'; -abstract class ParsedFile {} +abstract class FileResult {} -class ParsedDartFile extends ParsedFile { +class ParsedDartFile extends FileResult { final LibraryElement library; final List declaredTables; final List declaredDaos; final List declaredDatabases; + Iterable get dbAccessors => + declaredDatabases.cast().followedBy(declaredDaos); + ParsedDartFile( {@required this.library, this.declaredTables = const [], @@ -21,7 +23,7 @@ class ParsedDartFile extends ParsedFile { this.declaredDatabases = const []}); } -class ParsedMoorFile extends ParsedFile { +class ParsedMoorFile extends FileResult { final ParseResult parseResult; MoorFile get parsedFile => parseResult.rootNode as MoorFile; final List declaredTables; diff --git a/moor_generator/lib/src/analyzer/runner/steps.dart b/moor_generator/lib/src/analyzer/runner/steps.dart new file mode 100644 index 00000000..3664a41a --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/steps.dart @@ -0,0 +1,174 @@ +import 'package:analyzer/dart/constant/value.dart'; +import 'package:analyzer/dart/element/element.dart'; +import 'package:analyzer/dart/element/type.dart'; +import 'package:moor/moor.dart'; +import 'package:moor_generator/src/analyzer/dart/parser.dart'; +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/analyzer/moor/inline_dart_resolver.dart'; +import 'package:moor_generator/src/analyzer/moor/parser.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/sql_parser.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; +import 'package:moor_generator/src/analyzer/runner/task.dart'; +import 'package:moor_generator/src/model/specified_db_classes.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:source_gen/source_gen.dart'; + +/// A [Step] performs actions for a [Task] on a single file. +abstract class Step { + final Task task; + final FoundFile file; + final ErrorSink errors = ErrorSink(); + + String get path => file.uri.path; + + Step(this.task, this.file); + + void reportError(MoorError error) => errors.report(error); +} + +/// Extracts the following information from a Dart file: +/// - [SpecifiedTable]s, which are read from Dart classes extending `Table`. +/// - [SpecifiedDatabase]s, which are read from `@UseMoor`-annotated classes +/// - [SpecifiedDao]s, which are read from `@UseDao`-annotated classes. +/// +/// Notably, this step does not analyze defined queries. +class ParseDartStep extends Step { + static const _tableTypeChecker = const TypeChecker.fromRuntime(Table); + static const _useMoorChecker = const TypeChecker.fromRuntime(UseMoor); + static const _useDaoChecker = const TypeChecker.fromRuntime(UseDao); + + final LibraryElement library; + + MoorDartParser _parser; + MoorDartParser get parser => _parser; + + final Map _tables = {}; + + ParseDartStep(Task task, FoundFile file, this.library) : super(task, file) { + _parser = MoorDartParser(this); + } + + Future parse() async { + final reader = LibraryReader(library); + final databases = []; + final daos = []; + + for (var declaredClass in reader.classes) { + if (_tableTypeChecker.isAssignableFrom(declaredClass)) { + await _parseTable(declaredClass); + } else { + for (var annotation in _useMoorChecker.annotationsOf(declaredClass)) { + final reader = ConstantReader(annotation); + databases.add(await parseDatabase(declaredClass, reader)); + } + + for (var annotation in _useDaoChecker.annotationsOf(declaredClass)) { + final reader = ConstantReader(annotation); + daos.add(await parseDao(declaredClass, reader)); + } + } + } + + return ParsedDartFile( + library: library, + declaredTables: _tables.values.toList(), + declaredDaos: daos, + declaredDatabases: databases, + ); + } + + Future _parseTable(ClassElement element) async { + if (!_tables.containsKey(element)) { + _tables[element] = await parser.parseTable(element); + } + return _tables[element]; + } + + /// Parses a [SpecifiedDatabase] from the [ClassElement] which was annotated + /// with `@UseMoor` and the [annotation] reader that reads the `@UseMoor` + /// annotation. + Future parseDatabase( + ClassElement element, ConstantReader annotation) { + return UseMoorParser(this).parseDatabase(element, annotation); + } + + /// Parses a [SpecifiedDao] from a class declaration that has a `UseDao` + /// [annotation]. + Future parseDao( + ClassElement element, ConstantReader annotation) { + return UseDaoParser(this).parseDao(element, annotation); + } + + /// Resolves a [SpecifiedTable] for the class of each [DartType] in [types]. + /// The [initializedBy] element should be the piece of code that caused the + /// parsing (e.g. the database class that is annotated with `@UseMoor`). This + /// will allow for more descriptive error messages. + Future> parseTables( + Iterable types, Element initializedBy) { + return Future.wait(types.map((type) { + if (!_tableTypeChecker.isAssignableFrom(type.element)) { + reportError(ErrorInDartCode( + severity: Severity.criticalError, + message: 'The type $type is not a moor table', + affectedElement: initializedBy, + )); + return null; + } else { + return _parseTable(type.element as ClassElement); + } + })).then((list) { + // only keep tables that were resolved successfully + return List.from(list.where((t) => t != null)); + }); + } + + List readDeclaredQueries(Map obj) { + return obj.entries.map((entry) { + final key = entry.key.toStringValue(); + final value = entry.key.toStringValue(); + + return DeclaredQuery(key, value); + }).toList(); + } +} + +class ParseMoorFile extends Step { + final String content; + final TypeMapper mapper = TypeMapper(); + /* late final */ InlineDartResolver inlineDartResolver; + + ParseMoorFile(Task task, FoundFile file, this.content) : super(task, file) { + inlineDartResolver = InlineDartResolver(this); + } + + Future parseFile() { + final parser = MoorParser(this); + return parser.parseAndAnalyze(); + } +} + +/// Analyzes the compiled queries found in a Dart file. +class AnalyzeDartStep extends Step { + AnalyzeDartStep(Task task, FoundFile file) : super(task, file); + + void analyze() { + final parseResult = file.currentResult as ParsedDartFile; + + for (var accessor in parseResult.dbAccessors) { + final transitivelyAvailable = accessor.resolvedImports + .where((file) => file.type == FileType.moor) + .map((file) => file.currentResult as ParsedMoorFile) + .expand((file) => file.declaredTables); + final availableTables = + accessor.tables.followedBy(transitivelyAvailable).toList(); + + final parser = SqlParser(this, availableTables, accessor.queries); + parser.parse(); + + accessor.resolvedQueries = parser.foundQueries; + } + } +} diff --git a/moor_generator/lib/src/analyzer/runner/task.dart b/moor_generator/lib/src/analyzer/runner/task.dart new file mode 100644 index 00000000..de48dbcf --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/task.dart @@ -0,0 +1,151 @@ +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/backends/backend.dart'; +import 'package:moor_generator/src/model/specified_db_classes.dart'; + +/// A task is used to fully parse and analyze files based on an input file. To +/// analyze that file, all transitive imports will have to be analyzed as well. +/// +/// Analyzing works in two steps: +/// 1. parsing and reading the structure: For each Dart file we encounter, we +/// read all `UseMoor` and `UseDao` structures. We also read all `Table` +/// classes defined in that file. +/// 2. analyzing: Now that we have the table and database structure available, +/// can use that to analyze sql queries for semantic errors. +/// +/// The results of parsing a set of files is stored in a [MoorSession]. +class Task { + final FoundFile input; + final MoorSession session; + final BackendTask backend; + + final Map _performedSteps = {}; + final List _unhandled = []; + + Task(this.session, this.input, this.backend); + + Future runTask() async { + // step 1: parse all files included by the input + _unhandled.clear(); + _unhandled.add(input); + while (_unhandled.isNotEmpty) { + await _parse(_unhandled.removeLast()); + } + + // step 2: resolve queries in the input + for (var file in _performedSteps.keys) { + await _analyze(file); + } + } + + Future _parse(FoundFile file) async { + final resolvedImports = {}; + if (file.state != FileState.dirty) { + // already parsed, nothing to do :) + return; + } + + switch (file.type) { + case FileType.moor: + final content = await backend.readMoor(file.uri); + final step = ParseMoorFile(this, file, content); + _performedSteps[file] = step; + + final parsed = await step.parseFile(); + file.currentResult = parsed; + + for (var import in parsed.parsedFile.imports) { + final found = session.resolve(file, import.importedFile); + if (!await backend.exists(found.uri)) { + step.reportError(ErrorInMoorFile( + span: import.importString.span, + severity: Severity.error, + message: 'File does not exist: ${import.importedFile}', + )); + } else { + resolvedImports.add(found); + } + } + break; + case FileType.dart: + final library = await backend.resolveDart(file.uri); + final step = ParseDartStep(this, file, library); + _performedSteps[file] = step; + + final parsed = await step.parse(); + file.currentResult = parsed; + + final daosAndDatabases = parsed.declaredDaos + .cast() + .followedBy(parsed.declaredDatabases); + + for (var accessor in daosAndDatabases) { + final resolvedForAccessor = []; + + for (var import in accessor.includes) { + final found = session.resolve(file, import); + if (!await backend.exists(found.uri)) { + step.reportError(ErrorInDartCode( + affectedElement: accessor.fromClass, + severity: Severity.error, + message: 'Include could not be resolved: $import', + )); + } else { + resolvedImports.add(found); + resolvedForAccessor.add(found); + } + } + + accessor.resolvedImports = resolvedForAccessor; + } + break; + default: + break; + } + + file.state = FileState.parsed; + session.fileGraph.setImports(file, resolvedImports.toList()); + _notifyFilesNeedWork(resolvedImports); + } + + Future _analyze(FoundFile file) async { + // skip if already analyzed. + if (file.state == FileState.analyzed) return; + + switch (file.type) { + case FileType.dart: + AnalyzeDartStep(this, file).analyze(); + break; + default: + break; + } + + file.state = FileState.analyzed; + } + + void _notifyFilesNeedWork(Iterable files) { + for (var file in files) { + if (!_performedSteps.containsKey(file) && !_unhandled.contains(file)) { + _unhandled.add(file); + } + } + } + + void printErrors() { + final foundErrors = + _performedSteps.values.expand((step) => step.errors.errors); + if (foundErrors.isNotEmpty) { + final log = backend.log; + + log.warning('There were some errors while running moor_generator on ' + '${backend.entrypoint}:'); + + for (var error in foundErrors) { + final printer = error.isError ? log.warning : log.info; + error.writeDescription(printer); + } + } + } +} diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index d9f4495c..34c6d45a 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -1,172 +1,43 @@ -import 'dart:async'; - -import 'package:analyzer/dart/constant/value.dart'; -import 'package:analyzer/dart/element/element.dart'; -import 'package:analyzer/dart/element/type.dart'; -import 'package:moor/moor.dart' show Table; -import 'package:moor_generator/src/analyzer/dart/parser.dart'; -import 'package:moor_generator/src/analyzer/errors.dart'; -import 'package:moor_generator/src/analyzer/moor/inline_dart_resolver.dart'; -import 'package:moor_generator/src/analyzer/moor/parser.dart'; -import 'package:moor_generator/src/analyzer/results.dart'; -import 'package:moor_generator/src/analyzer/sql_queries/sql_parser.dart'; -import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/task.dart'; import 'package:moor_generator/src/backends/backend.dart'; -import 'package:moor_generator/src/model/specified_dao.dart'; -import 'package:moor_generator/src/model/specified_database.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/model/sql_query.dart'; -import 'package:source_gen/source_gen.dart'; +import 'package:path/path.dart' as p; + +const _fileEndings = { + '.moor': FileType.moor, + '.dart': FileType.dart, +}; /// Will store cached data about files that have already been analyzed. class MoorSession { - MoorSession(); + final FileGraph fileGraph = FileGraph(); + final Backend backend; - Future startDartTask(BackendTask backendTask, {Uri uri}) async { - final input = uri ?? backendTask.entrypoint; - final library = await backendTask.resolveDart(input); - return DartTask(this, backendTask, library); + MoorSession(this.backend); + + FileType _findFileType(String path) { + final extension = p.extension(path); + + return _fileEndings[extension] ?? FileType.other; } - Future startMoorTask(BackendTask backendTask, {Uri uri}) async { - final input = uri ?? backendTask.entrypoint; - final source = await backendTask.readMoor(input); - return MoorTask(backendTask, this, source); - } -} - -/// Used to parse and analyze a single file. -abstract class FileTask { - final BackendTask backendTask; - final MoorSession session; - - final ErrorSink errors = ErrorSink(); - - String get path => backendTask.entrypoint.path; - - FileTask(this.backendTask, this.session); - - void reportError(MoorError error) => errors.report(error); - - FutureOr compute(); - - void printErrors() { - final foundErrors = errors.errors; - if (foundErrors.isNotEmpty) { - final log = backendTask.log; - - log.warning('There were some errors while running ' - 'moor_generator on ${backendTask.entrypoint}:'); - - for (var error in foundErrors) { - final printer = error.isError ? log.warning : log.info; - error.writeDescription(printer); - } - } - } -} - -/// Session used to parse a Dart file and extract table information. -class DartTask extends FileTask { - static const tableTypeChecker = const TypeChecker.fromRuntime(Table); - - final LibraryElement library; - MoorDartParser _parser; - MoorDartParser get parser => _parser; - - DartTask(MoorSession session, BackendTask task, this.library) - : super(task, session) { - _parser = MoorDartParser(this); + /// Resolves an import directive in the context of the [source] file. This + /// can handle both relative imports and `package:` imports. + FoundFile resolve(FoundFile source, String import) { + final resolvedUri = backend.resolve(source.uri, import); + return _uriToFile(resolvedUri); } - @override - FutureOr compute() { - // TODO: implement compute - return null; - } + /// Registers a file by its absolute uri. + FoundFile registerFile(Uri file) => _uriToFile(file); - /// Parses a [SpecifiedDatabase] from the [ClassElement] which was annotated - /// with `@UseMoor` and the [annotation] reader that reads the `@UseMoor` - /// annotation. - Future parseDatabase( - ClassElement element, ConstantReader annotation) { - return UseMoorParser(this).parseDatabase(element, annotation); - } - - /// Parses a [SpecifiedDao] from a class declaration that has a `UseDao` - /// [annotation]. - Future parseDao( - ClassElement element, ConstantReader annotation) { - return UseDaoParser(this).parseDao(element, annotation); - } - - /// Resolves a [SpecifiedTable] for the class of each [DartType] in [types]. - /// The [initializedBy] element should be the piece of code that caused the - /// parsing (e.g. the database class that is annotated with `@UseMoor`). This - /// will allow for more descriptive error messages. - Future> parseTables( - Iterable types, Element initializedBy) { - return Future.wait(types.map((type) { - if (!tableTypeChecker.isAssignableFrom(type.element)) { - reportError(ErrorInDartCode( - severity: Severity.criticalError, - message: 'The type $type is not a moor table', - affectedElement: initializedBy, - )); - return null; - } else { - return parser.parseTable(type.element as ClassElement); - } - })).then((list) { - // only keep tables that were resolved successfully - return List.from(list.where((t) => t != null)); + FoundFile _uriToFile(Uri uri) { + return fileGraph.registerFile(uri, () { + return FoundFile(uri, _findFileType(uri.path)); }); } - /// Reads all tables declared in sql by a `.moor` file in [paths]. - Future> resolveIncludes(Iterable paths) { - return Stream.fromFutures(paths.map( - (path) => session.startMoorTask(backendTask, uri: Uri.parse(path)))) - .asyncMap((task) async { - final result = await task.compute(); - - // add errors from nested task to this task as well. - task.errors.errors.forEach(reportError); - - return result; - }) - .expand((file) => file.declaredTables) - .toList(); - } - - Future> parseQueries( - Map fromAnnotation, - List availableTables) { - // no queries declared, so there is no point in starting a sql engine - if (fromAnnotation.isEmpty) return Future.value([]); - - final parser = SqlParser(this, availableTables, fromAnnotation)..parse(); - - return Future.value(parser.foundQueries); - } -} - -class MoorTask extends FileTask { - final String content; - final TypeMapper mapper = TypeMapper(); - /* late final */ InlineDartResolver inlineDartResolver; - - ParsedMoorFile _lastResult; - ParsedMoorFile get lastResult => _lastResult; - - MoorTask(BackendTask task, MoorSession session, this.content) - : super(task, session) { - inlineDartResolver = InlineDartResolver(this); - } - - @override - FutureOr compute() { - final parser = MoorParser(this); - return parser.parseAndAnalyze().then((val) => _lastResult = val); + Task startTask(BackendTask backend) { + return Task(this, _uriToFile(backend.entrypoint), backend); } } diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index c1454660..b989aaf0 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -1,7 +1,6 @@ -import 'package:analyzer/dart/constant/value.dart'; import 'package:build/build.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; import 'package:moor_generator/src/analyzer/sql_queries/query_handler.dart'; @@ -10,15 +9,15 @@ import 'package:sqlparser/sqlparser.dart' hide ResultColumn; class SqlParser { final List tables; - final FileTask task; - final Map definedQueries; + final AnalyzeDartStep step; + final List definedQueries; final TypeMapper _mapper = TypeMapper(); SqlEngine _engine; final List foundQueries = []; - SqlParser(this.task, this.tables, this.definedQueries); + SqlParser(this.step, this.tables, this.definedQueries); void _spawnEngine() { _engine = SqlEngine(); @@ -28,39 +27,39 @@ class SqlParser { void parse() { _spawnEngine(); - definedQueries.forEach((key, value) { - final name = key.toStringValue(); - final sql = value.toStringValue(); + for (var query in definedQueries) { + final name = query.name; + final sql = query.sql; AnalysisContext context; try { context = _engine.analyze(sql); } catch (e, s) { - task.reportError(MoorError( + step.reportError(MoorError( severity: Severity.criticalError, - message: 'Error while trying to parse $key: $e, $s')); + message: 'Error while trying to parse $name: $e, $s')); return; } for (var error in context.errors) { - task.reportError(MoorError( + step.reportError(MoorError( severity: Severity.warning, - message: 'The sql query $key is invalid: $error', + message: 'The sql query $name is invalid: $error', )); } try { foundQueries.add(QueryHandler(name, context, _mapper).handle()); } catch (e, s) { - log.warning('Error while generating APIs for $key', e, s); + log.warning('Error while generating APIs for $name', e, s); } - }); + } // report lints for (var query in foundQueries) { for (var lint in query.lints) { - task.reportError(MoorError( - severity: Severity.warning, + step.reportError(MoorError( + severity: Severity.info, message: 'Lint for ${query.name}: $lint', )); } diff --git a/moor_generator/lib/src/backends/backend.dart b/moor_generator/lib/src/backends/backend.dart index 9b454f2d..ca74f3f7 100644 --- a/moor_generator/lib/src/backends/backend.dart +++ b/moor_generator/lib/src/backends/backend.dart @@ -8,7 +8,16 @@ import 'package:moor_generator/src/analyzer/session.dart'; /// Currently, we only have a backend based on the build package, but we can /// extend this to a backend for an analyzer plugin or a standalone tool. abstract class Backend { - final MoorSession session = MoorSession(); + MoorSession _session; + MoorSession get session => _session; + + Backend() { + _session = MoorSession(this); + } + + /// Resolves an [import] statement from the context of a [base] uri. This + /// should support both relative and `package:` imports. + Uri resolve(Uri base, String import); } /// Used to analyze a single file via ([entrypoint]). The other methods can be @@ -20,4 +29,14 @@ abstract class BackendTask { Future resolveDart(Uri uri); Future parseSource(String dart); Future readMoor(Uri uri); + + /// Checks whether a file at [uri] exists. + Future exists(Uri uri); + + /// Used from the higher-level api to notify the backend that a file would + /// have been read, but hasn't due to caching. + /// + /// We use this so that the build package can generate the dependency graph + /// correctly. + Future fakeRead(Uri uri) async {} } diff --git a/moor_generator/lib/src/backends/build/build_backend.dart b/moor_generator/lib/src/backends/build/build_backend.dart index 6fa105ea..09419fdc 100644 --- a/moor_generator/lib/src/backends/build/build_backend.dart +++ b/moor_generator/lib/src/backends/build/build_backend.dart @@ -3,18 +3,26 @@ import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart' hide log; import 'package:build/build.dart' as build show log; import 'package:logging/logging.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/backends/backend.dart'; class BuildBackend extends Backend { BuildBackendTask createTask(BuildStep step) { - return BuildBackendTask(step); + return BuildBackendTask(step, this); + } + + @override + Uri resolve(Uri base, String import) { + final from = AssetId.resolve(base.toString()); + return AssetId.resolve(import, from: from).uri; } } class BuildBackendTask extends BackendTask { final BuildStep step; + final BuildBackend backend; - BuildBackendTask(this.step); + BuildBackendTask(this.step, this.backend); @override Uri get entrypoint => step.inputId.uri; @@ -40,4 +48,18 @@ class BuildBackendTask extends BackendTask { @override Logger get log => build.log; + + @override + Future exists(Uri uri) { + return step.canRead(_resolve(uri)); + } + + Future finish(FoundFile inputFile) async { + // the result could be cached if it was calculated in a previous build step. + // we need to can canRead so that the build package can calculate the + // dependency graph correctly + for (var transitiveImport in backend.session.fileGraph.crawl(inputFile)) { + await step.canRead(_resolve(transitiveImport.uri)); + } + } } diff --git a/moor_generator/lib/src/backends/build/generators/dao_generator.dart b/moor_generator/lib/src/backends/build/generators/dao_generator.dart index 2abaf0e7..fb1a672c 100644 --- a/moor_generator/lib/src/backends/build/generators/dao_generator.dart +++ b/moor_generator/lib/src/backends/build/generators/dao_generator.dart @@ -1,68 +1,42 @@ -import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart'; -import 'package:moor/moor.dart'; import 'package:moor_generator/src/backends/build/moor_builder.dart'; import 'package:moor_generator/src/writer/queries/query_writer.dart'; -import 'package:moor_generator/src/writer/writer.dart'; import 'package:source_gen/source_gen.dart'; -class DaoGenerator extends GeneratorForAnnotation - implements BaseGenerator { +class DaoGenerator extends Generator implements BaseGenerator { @override MoorBuilder builder; @override - generateForAnnotatedElement( - Element element, ConstantReader annotation, BuildStep buildStep) async { - final task = await builder.createDartTask(buildStep); + Future generate(LibraryReader library, BuildStep buildStep) async { + final parsed = await builder.analyzeDartFile(buildStep); + final writer = builder.createWriter(); - if (element is! ClassElement) { - throw InvalidGenerationSourceError( - 'This annotation can only be used on classes', - element: element); + for (var dao in parsed.declaredDaos) { + final classScope = writer.child(); + final element = dao.fromClass; + + final daoName = element.displayName; + + classScope.leaf().write('mixin _\$${daoName}Mixin on ' + 'DatabaseAccessor<${dao.dbClass.displayName}> {\n'); + + for (var table in dao.tables) { + final infoType = table.tableInfoName; + final getterName = table.tableFieldName; + classScope + .leaf() + .write('$infoType get $getterName => db.$getterName;\n'); + } + + final writtenMappingMethods = {}; + for (var query in dao.resolvedQueries) { + QueryWriter(query, classScope.child(), writtenMappingMethods).write(); + } + + classScope.leaf().write('}'); } - final targetClass = element as ClassElement; - final parsedDao = await task.parseDao(targetClass, annotation); - - final dbType = targetClass.supertype; - if (dbType.name != 'DatabaseAccessor') { - throw InvalidGenerationSourceError( - 'This class must directly inherit from DatabaseAccessor', - element: element); - } - - // inherits from DatabaseAccessor, we want to know which T - final dbImpl = dbType.typeArguments.single; - if (dbImpl.isDynamic) { - throw InvalidGenerationSourceError( - 'This class must inherit from DatabaseAccessor, where T is an ' - 'actual type of a database.', - element: element); - } - - // finally, we can write the mixin - final writer = Writer(builder.options); - final classScope = writer.child(); - - final daoName = targetClass.displayName; - - classScope.leaf().write('mixin _\$${daoName}Mixin on ' - 'DatabaseAccessor<${dbImpl.displayName}> {\n'); - - for (var table in parsedDao.tables) { - final infoType = table.tableInfoName; - final getterName = table.tableFieldName; - classScope.leaf().write('$infoType get $getterName => db.$getterName;\n'); - } - - final writtenMappingMethods = {}; - for (var query in parsedDao.queries) { - QueryWriter(query, classScope.child(), writtenMappingMethods).write(); - } - - classScope.leaf().write('}'); - return writer.writeGenerated(); } } diff --git a/moor_generator/lib/src/backends/build/generators/moor_generator.dart b/moor_generator/lib/src/backends/build/generators/moor_generator.dart index 6adc3867..6f08ce18 100644 --- a/moor_generator/lib/src/backends/build/generators/moor_generator.dart +++ b/moor_generator/lib/src/backends/build/generators/moor_generator.dart @@ -1,43 +1,26 @@ -import 'package:moor/moor.dart'; -import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart'; -import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/backends/build/moor_builder.dart'; import 'package:moor_generator/src/writer/database_writer.dart'; -import 'package:moor_generator/src/writer/writer.dart'; import 'package:source_gen/source_gen.dart'; -class MoorGenerator extends GeneratorForAnnotation - implements BaseGenerator { +class MoorGenerator extends Generator implements BaseGenerator { @override MoorBuilder builder; @override - generateForAnnotatedElement( - Element element, ConstantReader annotation, BuildStep buildStep) async { - final task = await builder.createDartTask(buildStep); + Future generate(LibraryReader library, BuildStep buildStep) async { + final parsed = await builder.analyzeDartFile(buildStep); + final writer = builder.createWriter(); - if (element is! ClassElement) { - task.reportError(ErrorInDartCode( - severity: Severity.criticalError, - message: 'This annotation can only be used on classes', - affectedElement: element, - )); + if (parsed.declaredDatabases.isNotEmpty) { + writer + .leaf() + .write('// ignore_for_file: unnecessary_brace_in_string_interps\n'); } - final database = - await task.parseDatabase(element as ClassElement, annotation); - - task.printErrors(); - - if (database.tables.isEmpty) return ''; - - final writer = Writer(builder.options); - writer - .leaf() - .write('// ignore_for_file: unnecessary_brace_in_string_interps\n'); - - DatabaseWriter(database, writer.child()).write(); + for (var db in parsed.declaredDatabases) { + DatabaseWriter(db, writer.child()).write(); + } return writer.writeGenerated(); } diff --git a/moor_generator/lib/src/backends/build/moor_builder.dart b/moor_generator/lib/src/backends/build/moor_builder.dart index 4ff994e5..80334b8b 100644 --- a/moor_generator/lib/src/backends/build/moor_builder.dart +++ b/moor_generator/lib/src/backends/build/moor_builder.dart @@ -1,16 +1,21 @@ import 'package:build/build.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; import 'package:moor_generator/src/backends/build/build_backend.dart'; import 'package:moor_generator/src/backends/build/generators/dao_generator.dart'; import 'package:moor_generator/src/backends/build/generators/moor_generator.dart'; +import 'package:moor_generator/src/writer/writer.dart'; import 'package:source_gen/source_gen.dart'; part 'options.dart'; +final backendResource = Resource(() => BuildBackend()); + class MoorBuilder extends SharedPartBuilder { - final BuildBackend backend = BuildBackend(); final MoorOptions options; + MoorBuilder._(List generators, String name, this.options) + : super(generators, name); + factory MoorBuilder(BuilderOptions options) { final parsedOptions = MoorOptions.fromBuilder(options.config); @@ -28,13 +33,22 @@ class MoorBuilder extends SharedPartBuilder { return builder; } - MoorBuilder._(List generators, String name, this.options) - : super(generators, name); + Writer createWriter() => Writer(options); - Future createDartTask(BuildStep step) async { + Future analyzeDartFile(BuildStep step) async { + final backend = await step.fetchResource(backendResource); final backendTask = backend.createTask(step); - return await backend.session - .startDartTask(backendTask, uri: step.inputId.uri); + final session = backend.session; + + final input = session.registerFile(step.inputId.uri); + final task = session.startTask(backendTask); + await task.runTask(); + + task.printErrors(); + + await backendTask.finish(input); + + return input.currentResult as ParsedDartFile; } } diff --git a/moor_generator/lib/src/model/specified_dao.dart b/moor_generator/lib/src/model/specified_dao.dart deleted file mode 100644 index dfdcceb4..00000000 --- a/moor_generator/lib/src/model/specified_dao.dart +++ /dev/null @@ -1,12 +0,0 @@ -import 'package:analyzer/dart/element/element.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/model/sql_query.dart'; - -/// Model generated from a class that is annotated with `UseDao`. -class SpecifiedDao { - final ClassElement fromClass; - final List tables; - final List queries; - - SpecifiedDao(this.fromClass, this.tables, this.queries); -} diff --git a/moor_generator/lib/src/model/specified_database.dart b/moor_generator/lib/src/model/specified_database.dart deleted file mode 100644 index 84ccf5fd..00000000 --- a/moor_generator/lib/src/model/specified_database.dart +++ /dev/null @@ -1,13 +0,0 @@ -import 'package:analyzer/dart/element/element.dart'; -import 'package:analyzer/dart/element/type.dart'; -import 'package:moor_generator/src/model/specified_table.dart'; -import 'package:moor_generator/src/model/sql_query.dart'; - -class SpecifiedDatabase { - final ClassElement fromClass; - final List tables; - final List daos; - final List queries; - - SpecifiedDatabase(this.fromClass, this.tables, this.daos, this.queries); -} diff --git a/moor_generator/lib/src/model/specified_db_classes.dart b/moor_generator/lib/src/model/specified_db_classes.dart new file mode 100644 index 00000000..9ec6e29b --- /dev/null +++ b/moor_generator/lib/src/model/specified_db_classes.dart @@ -0,0 +1,40 @@ +import 'package:analyzer/dart/element/element.dart'; +import 'package:analyzer/dart/element/type.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; + +class SpecifiedDbAccessor { + final ClassElement fromClass; + + final List tables; + final List includes; + final List queries; + + List resolvedImports = []; + List resolvedQueries = const []; + + SpecifiedDbAccessor(this.fromClass, this.tables, this.includes, this.queries); +} + +/// Model generated from a class that is annotated with `UseDao`. +class SpecifiedDao extends SpecifiedDbAccessor { + /// The database class this dao belongs to. + final DartType dbClass; + + SpecifiedDao( + ClassElement fromClass, + this.dbClass, + List tables, + List includes, + List queries) + : super(fromClass, tables, includes, queries); +} + +class SpecifiedDatabase extends SpecifiedDbAccessor { + final List daos; + + SpecifiedDatabase(ClassElement fromClass, List tables, + this.daos, List includes, List queries) + : super(fromClass, tables, includes, queries); +} diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 75ad0f8e..8d596ace 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -7,6 +7,13 @@ import 'package:sqlparser/sqlparser.dart'; final _illegalChars = RegExp(r'[^0-9a-zA-Z_]'); final _leadingDigits = RegExp(r'^\d*'); +class DeclaredQuery { + final String name; + final String sql; + + DeclaredQuery(this.name, this.sql); +} + abstract class SqlQuery { final String name; final AnalysisContext fromContext; diff --git a/moor_generator/lib/src/writer/database_writer.dart b/moor_generator/lib/src/writer/database_writer.dart index ef9a9784..b03d26f3 100644 --- a/moor_generator/lib/src/writer/database_writer.dart +++ b/moor_generator/lib/src/writer/database_writer.dart @@ -1,9 +1,9 @@ +import 'package:moor_generator/src/model/specified_db_classes.dart'; import 'package:moor_generator/src/writer/queries/query_writer.dart'; import 'package:moor_generator/src/writer/tables/table_writer.dart'; import 'package:moor_generator/src/writer/utils/memoized_getter.dart'; import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; -import 'package:moor_generator/src/model/specified_database.dart'; class DatabaseWriter { final SpecifiedDatabase db; @@ -55,7 +55,7 @@ class DatabaseWriter { // Write implementation for query methods final writtenMappingMethods = {}; - for (var query in db.queries) { + for (var query in db.resolvedQueries) { QueryWriter(query, dbScope.child(), writtenMappingMethods).write(); } From b8cca3dcc2d5764d2e2a640701567703e016ed4e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 12 Sep 2019 16:01:28 +0200 Subject: [PATCH 068/117] Migrate AS plugin and build backend to new task api --- moor_generator/lib/src/analyzer/errors.dart | 14 +++ .../lib/src/analyzer/runner/file_graph.dart | 5 +- .../lib/src/analyzer/runner/steps.dart | 9 +- .../lib/src/analyzer/runner/task.dart | 16 ++- moor_generator/lib/src/analyzer/session.dart | 27 +++++ .../build/generators/dao_generator.dart | 2 +- .../src/backends/plugin/backend/driver.dart | 67 ++++++----- .../backends/plugin/backend/file_tracker.dart | 110 ++++++------------ .../plugin/backend/plugin_backend.dart | 16 ++- .../lib/src/backends/plugin/plugin.dart | 25 ++-- .../services/assists/assist_service.dart | 13 ++- .../plugin/services/autocomplete.dart | 33 +++--- .../src/backends/plugin/services/errors.dart | 44 ++++--- .../src/backends/plugin/services/folding.dart | 7 +- .../backends/plugin/services/highlights.dart | 19 +-- .../src/backends/plugin/services/outline.dart | 7 +- .../backends/plugin/services/requests.dart | 42 ++++--- .../lib/src/model/specified_db_classes.dart | 4 + .../lib/src/writer/database_writer.dart | 2 +- .../test/analyzer/dart/dart_test.dart | 14 ++- .../test/analyzer/dart/table_parser_test.dart | 16 +-- .../test/parser/moor/moor_parser_test.dart | 10 +- moor_generator/test/utils/test_backend.dart | 11 ++ 23 files changed, 310 insertions(+), 203 deletions(-) diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index 35998536..863ce41a 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -12,6 +12,8 @@ class MoorError { final Severity severity; final String message; + bool wasDuringParsing = true; + MoorError({@required this.severity, this.message}); bool get isError => @@ -69,6 +71,18 @@ class ErrorSink { void report(MoorError error) { _errors.add(error); } + + void clearAll() { + _errors.clear(); + } + + void clearNonParsingErrors() { + _errors.removeWhere((e) => !e.wasDuringParsing); + } + + void consume(ErrorSink other) { + _errors.addAll(other._errors); + } } enum Severity { diff --git a/moor_generator/lib/src/analyzer/runner/file_graph.dart b/moor_generator/lib/src/analyzer/runner/file_graph.dart index db722e05..8ab890f1 100644 --- a/moor_generator/lib/src/analyzer/runner/file_graph.dart +++ b/moor_generator/lib/src/analyzer/runner/file_graph.dart @@ -101,9 +101,12 @@ class FoundFile { FileState state; ErrorSink errors; + FoundFile(this.uri, this.type); + String get shortName => uri.pathSegments.last; - FoundFile(this.uri, this.type); + bool get isParsed => state != FileState.dirty; + bool get isAnalyzed => state == FileState.analyzed; @override int get hashCode => uri.hashCode; diff --git a/moor_generator/lib/src/analyzer/runner/steps.dart b/moor_generator/lib/src/analyzer/runner/steps.dart index 3664a41a..e0b824e5 100644 --- a/moor_generator/lib/src/analyzer/runner/steps.dart +++ b/moor_generator/lib/src/analyzer/runner/steps.dart @@ -22,11 +22,14 @@ abstract class Step { final FoundFile file; final ErrorSink errors = ErrorSink(); + bool get isParsing => true; + String get path => file.uri.path; Step(this.task, this.file); - void reportError(MoorError error) => errors.report(error); + void reportError(MoorError error) => + errors.report(error..wasDuringParsing = isParsing); } /// Extracts the following information from a Dart file: @@ -154,6 +157,9 @@ class ParseMoorFile extends Step { class AnalyzeDartStep extends Step { AnalyzeDartStep(Task task, FoundFile file) : super(task, file); + @override + final bool isParsing = false; + void analyze() { final parseResult = file.currentResult as ParsedDartFile; @@ -164,6 +170,7 @@ class AnalyzeDartStep extends Step { .expand((file) => file.declaredTables); final availableTables = accessor.tables.followedBy(transitivelyAvailable).toList(); + accessor.allTables = availableTables; final parser = SqlParser(this, availableTables, accessor.queries); parser.parse(); diff --git a/moor_generator/lib/src/analyzer/runner/task.dart b/moor_generator/lib/src/analyzer/runner/task.dart index de48dbcf..05064b88 100644 --- a/moor_generator/lib/src/analyzer/runner/task.dart +++ b/moor_generator/lib/src/analyzer/runner/task.dart @@ -26,6 +26,9 @@ class Task { Task(this.session, this.input, this.backend); + /// Returns an iterable of [FoundFile]s that were analyzed by this task. + Iterable get analyzedFiles => _performedSteps.keys; + Future runTask() async { // step 1: parse all files included by the input _unhandled.clear(); @@ -36,17 +39,23 @@ class Task { // step 2: resolve queries in the input for (var file in _performedSteps.keys) { + file.errors.clearNonParsingErrors(); await _analyze(file); } + + session.notifyTaskFinished(this); } Future _parse(FoundFile file) async { - final resolvedImports = {}; if (file.state != FileState.dirty) { // already parsed, nothing to do :) return; } + final resolvedImports = {}; + + file.errors.clearAll(); + switch (file.type) { case FileType.moor: final content = await backend.readMoor(file.uri); @@ -67,6 +76,7 @@ class Task { } else { resolvedImports.add(found); } + file.errors.consume(step.errors); } break; case FileType.dart: @@ -99,6 +109,7 @@ class Task { } accessor.resolvedImports = resolvedForAccessor; + file.errors.consume(step.errors); } break; default: @@ -116,7 +127,8 @@ class Task { switch (file.type) { case FileType.dart: - AnalyzeDartStep(this, file).analyze(); + final step = AnalyzeDartStep(this, file)..analyze(); + file.errors.consume(step.errors); break; default: break; diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 34c6d45a..939f909a 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -1,3 +1,5 @@ +import 'dart:async'; + import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/analyzer/runner/task.dart'; import 'package:moor_generator/src/backends/backend.dart'; @@ -13,8 +15,14 @@ class MoorSession { final FileGraph fileGraph = FileGraph(); final Backend backend; + final _completedTasks = StreamController.broadcast(); + final _changedFiles = StreamController.broadcast(); + MoorSession(this.backend); + /// Stream that emits a [Task] that has been completed. + Stream get completedTasks => _completedTasks.stream; + FileType _findFileType(String path) { final extension = p.extension(path); @@ -40,4 +48,23 @@ class MoorSession { Task startTask(BackendTask backend) { return Task(this, _uriToFile(backend.entrypoint), backend); } + + /// Notifies this backend that the content of the given [file] has been + /// changed. + void notifyFileChanged(FoundFile file) { + file.state = FileState.dirty; + // all files that transitively imported this files are no longer analyzed + // because they depend on this file. They're still parsed though + for (var affected in fileGraph.crawl(file, transposed: true)) { + if (affected.state == FileState.analyzed) { + affected.state = FileState.parsed; + } + } + + _changedFiles.add(file); + } + + void notifyTaskFinished(Task task) { + _completedTasks.add(task); + } } diff --git a/moor_generator/lib/src/backends/build/generators/dao_generator.dart b/moor_generator/lib/src/backends/build/generators/dao_generator.dart index fb1a672c..ea23c267 100644 --- a/moor_generator/lib/src/backends/build/generators/dao_generator.dart +++ b/moor_generator/lib/src/backends/build/generators/dao_generator.dart @@ -21,7 +21,7 @@ class DaoGenerator extends Generator implements BaseGenerator { classScope.leaf().write('mixin _\$${daoName}Mixin on ' 'DatabaseAccessor<${dao.dbClass.displayName}> {\n'); - for (var table in dao.tables) { + for (var table in dao.allTables) { final infoType = table.tableInfoName; final getterName = table.tableFieldName; classScope diff --git a/moor_generator/lib/src/backends/plugin/backend/driver.dart b/moor_generator/lib/src/backends/plugin/backend/driver.dart index bb16f51e..8981b011 100644 --- a/moor_generator/lib/src/backends/plugin/backend/driver.dart +++ b/moor_generator/lib/src/backends/plugin/backend/driver.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'package:analyzer/file_system/file_system.dart'; import 'package:analyzer/src/dart/analysis/file_state.dart'; import 'package:analyzer/src/dart/analysis/driver.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; import 'package:moor_generator/src/backends/plugin/backend/plugin_backend.dart'; @@ -19,19 +20,26 @@ class MoorDriver implements AnalysisDriverGeneric { final FileContentOverlay contentOverlay; final ResourceProvider _resourceProvider; - final MoorSession session = MoorSession(); + /* late final */ MoorSession session; + bool _isWorking = false; MoorDriver(this._tracker, this._scheduler, this.dartDriver, this.contentOverlay, this._resourceProvider) { _scheduler.add(this); + final backend = PluginBackend(this); + session = backend.session; } bool _ownsFile(String path) => path.endsWith('.moor'); + FoundFile pathToFoundFile(String path) { + return session.registerFile(Uri.parse(path)); + } + @override void addFile(String path) { if (_ownsFile(path)) { - _tracker.addFile(path); + pathToFoundFile(path); // will be registered if it doesn't exists } } @@ -44,7 +52,7 @@ class MoorDriver implements AnalysisDriverGeneric { void handleFileChanged(String path) { if (_ownsFile(path)) { - _tracker.handleContentChanged(path); + session.notifyFileChanged(pathToFoundFile(path)); _scheduler.notify(this); } } @@ -54,22 +62,18 @@ class MoorDriver implements AnalysisDriverGeneric { @override Future performWork() async { - final completer = Completer(); + if (_isWorking) return; - if (_tracker.hasWork) { - _tracker.work((path) async { - try { - final backendTask = _createTask(path); - final moorTask = await session.startMoorTask(backendTask); - await moorTask.compute(); + _isWorking = true; - return moorTask; - } finally { - completer.complete(); - } - }); + try { + final mostImportantFile = _tracker.fileWithHighestPriority; + final backendTask = _createTask(mostImportantFile.file.uri.path); - await completer.future; + final task = session.startTask(backendTask); + await task.runTask(); + } finally { + _isWorking = false; } } @@ -83,6 +87,11 @@ class MoorDriver implements AnalysisDriverGeneric { return file.exists ? file.readAsStringSync() : ''; } + bool doesFileExist(String path) { + return contentOverlay[path] != null || + _resourceProvider.getFile(path).exists; + } + /// Finds the absolute path of a [reference] url, optionally assuming that the /// [reference] appears in [base]. This supports both "package:"-based uris /// and relative imports. @@ -102,29 +111,31 @@ class MoorDriver implements AnalysisDriverGeneric { @override set priorityFiles(List priorityPaths) { - _tracker.setPriorityFiles(priorityPaths.where(_ownsFile)); + final found = priorityPaths.where(_ownsFile).map(pathToFoundFile); + _tracker.setPriorityFiles(found); } @override AnalysisDriverPriority get workPriority { if (_tracker.hasWork) { final mostImportant = _tracker.fileWithHighestPriority; - switch (mostImportant.currentPriority) { - case FilePriority.ignore: - return AnalysisDriverPriority.nothing; - case FilePriority.regular: - return AnalysisDriverPriority.general; - case FilePriority.interactive: - return AnalysisDriverPriority.interactive; - } + return mostImportant.currentPriority; } else { return AnalysisDriverPriority.nothing; } - throw AssertionError('unreachable'); } - Future parseMoorFile(String path) { + Stream completedFiles() { + return session.completedTasks.expand((task) => task.analyzedFiles); + } + + /// Waits for the file at [path] to be parsed. + Future waitFileParsed(String path) { _scheduler.notify(this); - return _tracker.results(path); + + final found = pathToFoundFile(path); + + return completedFiles() + .firstWhere((file) => file == found && file.isParsed); } } diff --git a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart index ef804f88..17cdcc9f 100644 --- a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart +++ b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart @@ -1,7 +1,10 @@ import 'dart:async'; +// ignore: implementation_imports +import 'package:analyzer/src/dart/analysis/driver.dart' + show AnalysisDriverPriority; import 'package:collection/collection.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; int _compareByPriority(TrackedFile a, TrackedFile b) { final aPriority = a.currentPriority?.index ?? 0; @@ -12,16 +15,12 @@ int _compareByPriority(TrackedFile a, TrackedFile b) { /// Keeps track of files that need to be analyzed by the moor plugin. class FileTracker { PriorityQueue _pendingWork; - final Map _trackedFiles = {}; + final Map _trackedFiles = {}; final Set _currentPriority = {}; final StreamController _computations = StreamController.broadcast(); - /// Streams that emits a [TrackedFile] when it has been worked on - /// successfully. - Stream get computations => _computations.stream; - FileTracker() { _pendingWork = PriorityQueue(_compareByPriority); } @@ -29,7 +28,12 @@ class FileTracker { void _updateFile(TrackedFile file, Function(TrackedFile) update) { _pendingWork.remove(file); update(file); - _pendingWork.add(file); + + // if a file is analyzed, we don't need to do anything. So don't add it to + // list of of tracked files. + if (!file.file.isAnalyzed) { + _pendingWork.add(file); + } } void _putInQueue(TrackedFile file) { @@ -41,102 +45,56 @@ class FileTracker { bool get hasWork => _pendingWork.isNotEmpty; TrackedFile get fileWithHighestPriority => _pendingWork.first; - TrackedFile addFile(String path) { - return _trackedFiles.putIfAbsent(path, () { - final tracked = TrackedFile(path); + void notifyAnalysisStateChanged(FoundFile file) { + _putInQueue(_addFile(file)); + } + + TrackedFile _addFile(FoundFile file) { + return _trackedFiles.putIfAbsent(file, () { + final tracked = TrackedFile(file); _pendingWork.add(tracked); return tracked; }); } - void handleContentChanged(String path) { - _putInQueue(addFile(path)); - } - - void setPriorityFiles(Iterable priority) { + void setPriorityFiles(Iterable priority) { // remove prioritized flag from existing files for (var file in _currentPriority) { _updateFile(file, (f) => f._prioritized = false); } _currentPriority ..clear() - ..addAll(priority.map(addFile)) + ..addAll(priority.map(_addFile)) ..forEach((file) { _updateFile(file, (f) => f._prioritized = true); }); } - void notifyFileChanged(String path) { - final tracked = addFile(path); - tracked._currentResult = null; - _putInQueue(tracked); - } - - Future results(String path) async { - final tracked = addFile(path); - - if (tracked._currentResult != null) { - return tracked._currentResult; - } else { - final completer = Completer(); - tracked._waiting.add(completer); - return completer.future; - } - } - - void work(Future Function(String path) worker) { - if (_pendingWork.isNotEmpty) { - final unit = _pendingWork.removeFirst(); - - worker(unit.path).then((result) { - _computations.add(unit); - - for (var completer in unit._waiting) { - completer.complete(result); - } - unit._waiting.clear(); - }, onError: (e, StackTrace s) { - for (var completer in unit._waiting) { - completer.completeError(e, s); - } - unit._waiting.clear(); - }); - } - } - void dispose() { _computations.close(); } } -enum FileType { moor, unknown } - -enum FilePriority { ignore, regular, interactive } - -const Map _defaultPrio = { - FileType.moor: FilePriority.regular, - FileType.unknown: FilePriority.ignore, -}; - class TrackedFile { - final String path; - final FileType type; + final FoundFile file; /// Whether this file has been given an elevated priority, for instance /// because the user is currently typing in it. bool _prioritized = false; - MoorTask _currentResult; - final List> _waiting = []; - FilePriority get currentPriority => - _prioritized ? FilePriority.interactive : defaultPriority; - - TrackedFile._(this.path, this.type); - - factory TrackedFile(String path) { - final type = path.endsWith('.moor') ? FileType.moor : FileType.unknown; - return TrackedFile._(path, type); + AnalysisDriverPriority get currentPriority { + if (_prioritized) { + return file.state == FileState.dirty + ? AnalysisDriverPriority.interactive + : AnalysisDriverPriority.priority; + } else if (file.state == FileState.analyzed) { + return AnalysisDriverPriority.general; + } else if (file.state == FileState.parsed) { + return AnalysisDriverPriority.generalImportChanged; + } else { + return AnalysisDriverPriority.changedFiles; + } } - FilePriority get defaultPriority => _defaultPrio[type]; + TrackedFile(this.file); } diff --git a/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart b/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart index bf55105d..75095805 100644 --- a/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart +++ b/moor_generator/lib/src/backends/plugin/backend/plugin_backend.dart @@ -5,7 +5,16 @@ import 'package:moor_generator/src/backends/backend.dart'; import 'driver.dart'; -class PluginBackend extends Backend {} +class PluginBackend extends Backend { + final MoorDriver driver; + + PluginBackend(this.driver); + + @override + Uri resolve(Uri base, String import) { + return Uri.parse(driver.absolutePath(Uri.parse(import), base: base)); + } +} class PluginTask extends BackendTask { @override @@ -33,4 +42,9 @@ class PluginTask extends BackendTask { final path = driver.absolutePath(uri, base: entrypoint); return driver.dartDriver.currentSession.getLibraryByUri(path); } + + @override + Future exists(Uri uri) { + return Future.value(driver.doesFileExist(uri.path)); + } } diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart index 73c29c28..6523c29c 100644 --- a/moor_generator/lib/src/backends/plugin/plugin.dart +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -70,12 +70,15 @@ class MoorPlugin extends ServerPlugin final tracker = FileTracker(); final errorService = ErrorService(this); - tracker.computations - .asyncMap((file) => tracker.results(file.path)) - .listen(errorService.handleMoorResult); - - return MoorDriver(tracker, analysisDriverScheduler, dartDriver, + final driver = MoorDriver(tracker, analysisDriverScheduler, dartDriver, fileContentOverlay, resourceProvider); + + driver + .completedFiles() + .where((file) => file.isParsed) + .listen(errorService.handleResult); + + return driver; } @override @@ -92,9 +95,9 @@ class MoorPlugin extends ServerPlugin Future _createMoorRequest(String path) async { final driver = _moorDriverForPath(path); - final task = await driver.parseMoorFile(path); + final file = await driver.waitFileParsed(path); - return MoorRequest(task, resourceProvider); + return MoorRequest(file, resourceProvider); } @override @@ -134,9 +137,9 @@ class MoorPlugin extends ServerPlugin plugin.CompletionGetSuggestionsParams parameters) async { final path = parameters.file; final driver = _moorDriverForPath(path); - final task = await driver.parseMoorFile(path); + final file = await driver.waitFileParsed(path); - return MoorCompletionRequest(parameters.offset, resourceProvider, task); + return MoorCompletionRequest(parameters.offset, resourceProvider, file); } @override @@ -149,9 +152,9 @@ class MoorPlugin extends ServerPlugin plugin.EditGetAssistsParams parameters) async { final path = parameters.file; final driver = _moorDriverForPath(path); - final task = await driver.parseMoorFile(path); + final file = await driver.waitFileParsed(path); return MoorAssistRequest( - task, parameters.length, parameters.offset, resourceProvider); + file, parameters.length, parameters.offset, resourceProvider); } } diff --git a/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart b/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart index bcd215fa..578750fe 100644 --- a/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart +++ b/moor_generator/lib/src/backends/plugin/services/assists/assist_service.dart @@ -14,12 +14,15 @@ class AssistService implements AssistContributor { @override void computeAssists(AssistRequest request, AssistCollector collector) { final moorRequest = request as MoorAssistRequest; - final parseResult = moorRequest.task.lastResult.parseResult; - final relevantNodes = - parseResult.findNodesAtPosition(request.offset, length: request.length); - for (var node in relevantNodes.expand((node) => node.selfAndParents)) { - _handleNode(collector, node, moorRequest.task.path); + if (moorRequest.isMoorAndParsed) { + final parseResult = moorRequest.parsedMoor.parseResult; + final relevantNodes = parseResult.findNodesAtPosition(request.offset, + length: request.length); + + for (var node in relevantNodes.expand((node) => node.selfAndParents)) { + _handleNode(collector, node, moorRequest.path); + } } } diff --git a/moor_generator/lib/src/backends/plugin/services/autocomplete.dart b/moor_generator/lib/src/backends/plugin/services/autocomplete.dart index 4bfab9a9..38f4edc3 100644 --- a/moor_generator/lib/src/backends/plugin/services/autocomplete.dart +++ b/moor_generator/lib/src/backends/plugin/services/autocomplete.dart @@ -8,25 +8,26 @@ class MoorCompletingContributor implements CompletionContributor { @override Future computeSuggestions( MoorCompletionRequest request, CompletionCollector collector) { - final autoComplete = request.task.lastResult.parseResult.autoCompleteEngine; - final results = autoComplete.suggestCompletions(request.offset); + if (request.isMoorAndParsed) { + final autoComplete = request.parsedMoor.parseResult.autoCompleteEngine; + final results = autoComplete.suggestCompletions(request.offset); - collector - ..offset = results.anchor - ..length = results.lengthBefore; + collector + ..offset = results.anchor + ..length = results.lengthBefore; - for (var suggestion in results.suggestions) { - collector.addSuggestion(CompletionSuggestion( - CompletionSuggestionKind.KEYWORD, - suggestion.relevance, - suggestion.code, - -1, - -1, - false, - false, - )); + for (var suggestion in results.suggestions) { + collector.addSuggestion(CompletionSuggestion( + CompletionSuggestionKind.KEYWORD, + suggestion.relevance, + suggestion.code, + -1, + -1, + false, + false, + )); + } } - return Future.value(); } } diff --git a/moor_generator/lib/src/backends/plugin/services/errors.dart b/moor_generator/lib/src/backends/plugin/services/errors.dart index 2e5aa067..ccbdf763 100644 --- a/moor_generator/lib/src/backends/plugin/services/errors.dart +++ b/moor_generator/lib/src/backends/plugin/services/errors.dart @@ -1,6 +1,7 @@ import 'package:analyzer_plugin/protocol/protocol_common.dart'; import 'package:analyzer_plugin/protocol/protocol_generated.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/backends/plugin/plugin.dart'; const _parsingErrorCode = 'moor.parsingError'; @@ -12,27 +13,40 @@ class ErrorService { ErrorService(this.plugin); - void handleMoorResult(MoorTask completedTask) { - final result = completedTask.lastResult.parseResult; - final path = completedTask.backendTask.entrypoint.path; - + void handleResult(FoundFile analyzedFile) { final errors = []; + final path = analyzedFile.uri.path; - for (var error in result.errors) { - // this is a parsing error, high severity - final severity = AnalysisErrorSeverity.ERROR; - final type = AnalysisErrorType.SYNTACTIC_ERROR; + if (analyzedFile.isParsed) { + for (var error in analyzedFile.errors.errors) { + // this is a parsing error, high severity + final severity = error.isError + ? AnalysisErrorSeverity.ERROR + : AnalysisErrorSeverity.WARNING; - final sourceSpan = error.token.span; - final start = sourceSpan.start; - final location = Location( - path, start.offset, sourceSpan.length, start.line, start.column); + final type = error.wasDuringParsing + ? AnalysisErrorType.SYNTACTIC_ERROR + : AnalysisErrorType.COMPILE_TIME_ERROR; - errors.add(AnalysisError( - severity, type, location, error.message, _parsingErrorCode)); + final location = _findLocationForError(error, path); + + errors.add(AnalysisError( + severity, type, location, error.message, _parsingErrorCode)); + } } final params = AnalysisErrorsParams(path, errors); plugin.channel.sendNotification(params.toNotification()); } + + Location _findLocationForError(MoorError error, String path) { + if (error is ErrorInMoorFile) { + final span = error.span; + final start = span.start; + return Location( + path, start.offset, span.length, start.line, start.column); + } + + return Location(path, -1, -1, 0, 0); + } } diff --git a/moor_generator/lib/src/backends/plugin/services/folding.dart b/moor_generator/lib/src/backends/plugin/services/folding.dart index f065f50b..f205b078 100644 --- a/moor_generator/lib/src/backends/plugin/services/folding.dart +++ b/moor_generator/lib/src/backends/plugin/services/folding.dart @@ -10,8 +10,11 @@ class MoorFoldingContributor implements FoldingContributor { void computeFolding(FoldingRequest request, FoldingCollector collector) { final moorRequest = request as MoorRequest; - final visitor = _FoldingVisitor(collector); - moorRequest.resolvedTask.lastResult.parsedFile.accept(visitor); + if (moorRequest.isMoorAndParsed) { + final result = moorRequest.parsedMoor; + final visitor = _FoldingVisitor(collector); + result.parsedFile.accept(visitor); + } } } diff --git a/moor_generator/lib/src/backends/plugin/services/highlights.dart b/moor_generator/lib/src/backends/plugin/services/highlights.dart index a3b8c690..87f35624 100644 --- a/moor_generator/lib/src/backends/plugin/services/highlights.dart +++ b/moor_generator/lib/src/backends/plugin/services/highlights.dart @@ -14,17 +14,18 @@ class MoorHighlightContributor implements HighlightsContributor { } final typedRequest = request as MoorRequest; - final visitor = _HighlightingVisitor(collector); + if (typedRequest.isMoorAndParsed) { + final result = typedRequest.parsedMoor; - final result = typedRequest.resolvedTask.lastResult; + final visitor = _HighlightingVisitor(collector); + result.parsedFile.accept(visitor); - result.parsedFile.accept(visitor); - - for (var token in result.parseResult.tokens) { - if (token is KeywordToken && !token.isIdentifier) { - final start = token.span.start.offset; - final length = token.span.length; - collector.addRegion(start, length, HighlightRegionType.BUILT_IN); + for (var token in result.parseResult.tokens) { + if (token is KeywordToken && !token.isIdentifier) { + final start = token.span.start.offset; + final length = token.span.length; + collector.addRegion(start, length, HighlightRegionType.BUILT_IN); + } } } } diff --git a/moor_generator/lib/src/backends/plugin/services/outline.dart b/moor_generator/lib/src/backends/plugin/services/outline.dart index 328cf5ea..05a5cef6 100644 --- a/moor_generator/lib/src/backends/plugin/services/outline.dart +++ b/moor_generator/lib/src/backends/plugin/services/outline.dart @@ -12,8 +12,11 @@ class MoorOutlineContributor implements OutlineContributor { void computeOutline(OutlineRequest request, OutlineCollector collector) { final moorRequest = request as MoorRequest; - final visitor = _OutlineVisitor(collector); - moorRequest.resolvedTask.lastResult.parsedFile.accept(visitor); + if (moorRequest.isMoorAndParsed) { + final visitor = _OutlineVisitor(collector); + + moorRequest.parsedMoor.parsedFile.accept(visitor); + } } } diff --git a/moor_generator/lib/src/backends/plugin/services/requests.dart b/moor_generator/lib/src/backends/plugin/services/requests.dart index 9a9d2c5b..11098e21 100644 --- a/moor_generator/lib/src/backends/plugin/services/requests.dart +++ b/moor_generator/lib/src/backends/plugin/services/requests.dart @@ -4,22 +4,36 @@ import 'package:analyzer_plugin/utilities/completion/completion_core.dart'; import 'package:analyzer_plugin/utilities/folding/folding.dart'; import 'package:analyzer_plugin/utilities/highlights/highlights.dart'; import 'package:analyzer_plugin/utilities/outline/outline.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; -class MoorRequest implements OutlineRequest, HighlightsRequest, FoldingRequest { - final MoorTask resolvedTask; +mixin _MoorBaseRequest { + FoundFile get file; + + bool get isMoorAndParsed => file.type == FileType.moor && file.isParsed; + + String get path => file.uri.path; + + ParsedMoorFile get parsedMoor { + assert(isMoorAndParsed); + return file.currentResult as ParsedMoorFile; + } +} + +class MoorRequest + with _MoorBaseRequest + implements OutlineRequest, HighlightsRequest, FoldingRequest { + @override + final FoundFile file; @override final ResourceProvider resourceProvider; - MoorRequest(this.resolvedTask, this.resourceProvider); - - @override - String get path => resolvedTask.backendTask.entrypoint.toFilePath(); + MoorRequest(this.file, this.resourceProvider); } // todo CompletionRequest likes not to be extended, but there is no suitable // subclass. -class MoorCompletionRequest extends CompletionRequest { +class MoorCompletionRequest extends CompletionRequest with _MoorBaseRequest { @override void checkAborted() {} @@ -29,13 +43,15 @@ class MoorCompletionRequest extends CompletionRequest { @override final ResourceProvider resourceProvider; - final MoorTask task; + @override + final FoundFile file; - MoorCompletionRequest(this.offset, this.resourceProvider, this.task); + MoorCompletionRequest(this.offset, this.resourceProvider, this.file); } -class MoorAssistRequest extends AssistRequest { - final MoorTask task; +class MoorAssistRequest extends AssistRequest with _MoorBaseRequest { + @override + final FoundFile file; @override final int length; @@ -46,5 +62,5 @@ class MoorAssistRequest extends AssistRequest { @override final ResourceProvider resourceProvider; - MoorAssistRequest(this.task, this.length, this.offset, this.resourceProvider); + MoorAssistRequest(this.file, this.length, this.offset, this.resourceProvider); } diff --git a/moor_generator/lib/src/model/specified_db_classes.dart b/moor_generator/lib/src/model/specified_db_classes.dart index 9ec6e29b..a581c9bf 100644 --- a/moor_generator/lib/src/model/specified_db_classes.dart +++ b/moor_generator/lib/src/model/specified_db_classes.dart @@ -14,6 +14,10 @@ class SpecifiedDbAccessor { List resolvedImports = []; List resolvedQueries = const []; + /// All tables available to this class. This includes the [tables] and all + /// tables defined in a [includes] table. + List allTables = []; + SpecifiedDbAccessor(this.fromClass, this.tables, this.includes, this.queries); } diff --git a/moor_generator/lib/src/writer/database_writer.dart b/moor_generator/lib/src/writer/database_writer.dart index b03d26f3..b2a920e1 100644 --- a/moor_generator/lib/src/writer/database_writer.dart +++ b/moor_generator/lib/src/writer/database_writer.dart @@ -13,7 +13,7 @@ class DatabaseWriter { void write() { // Write referenced tables - for (final table in db.tables) { + for (final table in db.allTables) { TableWriter(table, scope.child()).writeInto(); } diff --git a/moor_generator/test/analyzer/dart/dart_test.dart b/moor_generator/test/analyzer/dart/dart_test.dart index b49de7ca..a73e275c 100644 --- a/moor_generator/test/analyzer/dart/dart_test.dart +++ b/moor_generator/test/analyzer/dart/dart_test.dart @@ -2,6 +2,7 @@ import 'package:analyzer/dart/ast/ast.dart'; import 'package:analyzer/dart/element/element.dart'; import 'package:build/build.dart'; import 'package:moor_generator/src/analyzer/dart/parser.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:test/test.dart'; import '../../utils/test_backend.dart'; @@ -20,10 +21,11 @@ void main() { ''' }); - final backendTask = - backend.startTask(Uri.parse('package:test_lib/main.dart')); - final dartTask = await backend.session.startDartTask(backendTask); - final parser = MoorDartParser(dartTask); + final input = Uri.parse('package:test_lib/main.dart'); + final backendTask = backend.startTask(input); + + final library = await backendTask.resolveDart(input); + final parser = MoorDartParser(ParseDartStep(null, null, library)); Future _loadDeclaration(Element element) async { final declaration = await parser.loadElementDeclaration(element); @@ -35,14 +37,14 @@ void main() { expect(parser.returnExpressionOfMethod(node).toSource(), source); } - final testClass = dartTask.library.getType('Test'); + final testClass = library.getType('Test'); _verifyReturnExpressionMatches(testClass.getGetter('getter'), "'foo'"); _verifyReturnExpressionMatches(testClass.getMethod('function'), "'bar'"); final invalidDecl = await _loadDeclaration(testClass.getMethod('invalid')); expect(parser.returnExpressionOfMethod(invalidDecl), isNull); - expect(dartTask.errors.errors, isNotEmpty); + expect(parser.step.errors.errors, isNotEmpty); backend.finish(); }); diff --git a/moor_generator/test/analyzer/dart/table_parser_test.dart b/moor_generator/test/analyzer/dart/table_parser_test.dart index dea23597..85a479c7 100644 --- a/moor_generator/test/analyzer/dart/table_parser_test.dart +++ b/moor_generator/test/analyzer/dart/table_parser_test.dart @@ -1,6 +1,6 @@ import 'package:build/build.dart'; import 'package:moor_generator/src/analyzer/dart/parser.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:test/test.dart'; @@ -9,7 +9,7 @@ import '../../utils/test_backend.dart'; void main() { TestBackend backend; - DartTask dartTask; + ParseDartStep dartStep; MoorDartParser parser; setUpAll(() { backend = TestBackend({ @@ -48,13 +48,15 @@ void main() { }); setUp(() async { - final task = backend.startTask(Uri.parse('package:test_lib/main.dart')); - dartTask = await backend.session.startDartTask(task); - parser = MoorDartParser(dartTask); + final uri = Uri.parse('package:test_lib/main.dart'); + final task = backend.startTask(uri); + + dartStep = ParseDartStep(null, null, await task.resolveDart(uri)); + parser = MoorDartParser(dartStep); }); Future parse(String name) async { - return parser.parseTable(dartTask.library.getType(name)); + return parser.parseTable(dartStep.library.getType(name)); } group('table names', () { @@ -70,7 +72,7 @@ void main() { test('should not parse for complex methods', () async { await parse('WrongName'); - expect(dartTask.errors.errors, isNotEmpty); + expect(dartStep.errors.errors, isNotEmpty); }); }); diff --git a/moor_generator/test/parser/moor/moor_parser_test.dart b/moor_generator/test/parser/moor/moor_parser_test.dart index 55242a0e..e4db549d 100644 --- a/moor_generator/test/parser/moor/moor_parser_test.dart +++ b/moor_generator/test/parser/moor/moor_parser_test.dart @@ -1,5 +1,4 @@ -import 'package:moor_generator/src/analyzer/moor/parser.dart'; -import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:test_api/test_api.dart'; void main() { @@ -11,11 +10,10 @@ CREATE TABLE users( '''; test('extracts table structure from .moor files', () async { - final task = MoorTask(null, null, content); - final analyzer = MoorParser(task); - final result = await analyzer.parseAndAnalyze(); + final parseStep = ParseMoorFile(null, null, content); + final result = await parseStep.parseFile(); - expect(task.errors.errors, isEmpty); + expect(parseStep.errors.errors, isEmpty); final table = result.declaredTables.single; diff --git a/moor_generator/test/utils/test_backend.dart b/moor_generator/test/utils/test_backend.dart index 3669e301..dfc8e41d 100644 --- a/moor_generator/test/utils/test_backend.dart +++ b/moor_generator/test/utils/test_backend.dart @@ -37,6 +37,12 @@ class TestBackend extends Backend { void finish() { _finish.complete(); } + + @override + Uri resolve(Uri base, String import) { + final from = AssetId.resolve(base.toString()); + return AssetId.resolve(import, from: from).uri; + } } class _TestBackendTask extends BackendTask { @@ -66,4 +72,9 @@ class _TestBackendTask extends BackendTask { Future parseSource(String dart) { return null; } + + @override + Future exists(Uri uri) async { + return backend.fakeContent.containsKey(AssetId.parse(uri.toString())); + } } From f3db52717ff9a50146777ec4cec0afc273715e01 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 12 Sep 2019 19:09:56 +0200 Subject: [PATCH 069/117] Extract steps into multiple files --- .../analyzer/moor/inline_dart_resolver.dart | 2 +- .../lib/src/analyzer/moor/parser.dart | 2 +- .../lib/src/analyzer/runner/file_graph.dart | 4 +- .../lib/src/analyzer/runner/steps.dart | 152 +----------------- .../analyzer/runner/steps/analyze_dart.dart | 28 ++++ .../src/analyzer/runner/steps/parse_dart.dart | 111 +++++++++++++ .../src/analyzer/runner/steps/parse_moor.dart | 16 ++ .../lib/src/analyzer/runner/task.dart | 49 +++--- .../lib/src/writer/database_writer.dart | 2 +- .../test/parser/moor/moor_parser_test.dart | 2 +- 10 files changed, 194 insertions(+), 174 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart create mode 100644 moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart create mode 100644 moor_generator/lib/src/analyzer/runner/steps/parse_moor.dart diff --git a/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart b/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart index e5774e2b..49a30e62 100644 --- a/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart +++ b/moor_generator/lib/src/analyzer/moor/inline_dart_resolver.dart @@ -18,7 +18,7 @@ import 'package:moor_generator/src/analyzer/runner/steps.dart'; /// of the top-level `expr` variable in that source. class InlineDartResolver { final List importStatements = []; - final ParseMoorFile step; + final ParseMoorStep step; InlineDartResolver(this.step); diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index 66e214ab..17ca7737 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -5,7 +5,7 @@ import 'package:moor_generator/src/analyzer/runner/results.dart'; import 'package:sqlparser/sqlparser.dart'; class MoorParser { - final ParseMoorFile step; + final ParseMoorStep step; MoorParser(this.step); diff --git a/moor_generator/lib/src/analyzer/runner/file_graph.dart b/moor_generator/lib/src/analyzer/runner/file_graph.dart index 8ab890f1..59b9aab9 100644 --- a/moor_generator/lib/src/analyzer/runner/file_graph.dart +++ b/moor_generator/lib/src/analyzer/runner/file_graph.dart @@ -98,8 +98,8 @@ class FoundFile { final FileType type; FileResult currentResult; - FileState state; - ErrorSink errors; + /* (not null) */ FileState state = FileState.dirty; + final ErrorSink errors = ErrorSink(); FoundFile(this.uri, this.type); diff --git a/moor_generator/lib/src/analyzer/runner/steps.dart b/moor_generator/lib/src/analyzer/runner/steps.dart index e0b824e5..9ed58f15 100644 --- a/moor_generator/lib/src/analyzer/runner/steps.dart +++ b/moor_generator/lib/src/analyzer/runner/steps.dart @@ -16,6 +16,10 @@ import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; import 'package:source_gen/source_gen.dart'; +part 'steps/analyze_dart.dart'; +part 'steps/parse_dart.dart'; +part 'steps/parse_moor.dart'; + /// A [Step] performs actions for a [Task] on a single file. abstract class Step { final Task task; @@ -31,151 +35,3 @@ abstract class Step { void reportError(MoorError error) => errors.report(error..wasDuringParsing = isParsing); } - -/// Extracts the following information from a Dart file: -/// - [SpecifiedTable]s, which are read from Dart classes extending `Table`. -/// - [SpecifiedDatabase]s, which are read from `@UseMoor`-annotated classes -/// - [SpecifiedDao]s, which are read from `@UseDao`-annotated classes. -/// -/// Notably, this step does not analyze defined queries. -class ParseDartStep extends Step { - static const _tableTypeChecker = const TypeChecker.fromRuntime(Table); - static const _useMoorChecker = const TypeChecker.fromRuntime(UseMoor); - static const _useDaoChecker = const TypeChecker.fromRuntime(UseDao); - - final LibraryElement library; - - MoorDartParser _parser; - MoorDartParser get parser => _parser; - - final Map _tables = {}; - - ParseDartStep(Task task, FoundFile file, this.library) : super(task, file) { - _parser = MoorDartParser(this); - } - - Future parse() async { - final reader = LibraryReader(library); - final databases = []; - final daos = []; - - for (var declaredClass in reader.classes) { - if (_tableTypeChecker.isAssignableFrom(declaredClass)) { - await _parseTable(declaredClass); - } else { - for (var annotation in _useMoorChecker.annotationsOf(declaredClass)) { - final reader = ConstantReader(annotation); - databases.add(await parseDatabase(declaredClass, reader)); - } - - for (var annotation in _useDaoChecker.annotationsOf(declaredClass)) { - final reader = ConstantReader(annotation); - daos.add(await parseDao(declaredClass, reader)); - } - } - } - - return ParsedDartFile( - library: library, - declaredTables: _tables.values.toList(), - declaredDaos: daos, - declaredDatabases: databases, - ); - } - - Future _parseTable(ClassElement element) async { - if (!_tables.containsKey(element)) { - _tables[element] = await parser.parseTable(element); - } - return _tables[element]; - } - - /// Parses a [SpecifiedDatabase] from the [ClassElement] which was annotated - /// with `@UseMoor` and the [annotation] reader that reads the `@UseMoor` - /// annotation. - Future parseDatabase( - ClassElement element, ConstantReader annotation) { - return UseMoorParser(this).parseDatabase(element, annotation); - } - - /// Parses a [SpecifiedDao] from a class declaration that has a `UseDao` - /// [annotation]. - Future parseDao( - ClassElement element, ConstantReader annotation) { - return UseDaoParser(this).parseDao(element, annotation); - } - - /// Resolves a [SpecifiedTable] for the class of each [DartType] in [types]. - /// The [initializedBy] element should be the piece of code that caused the - /// parsing (e.g. the database class that is annotated with `@UseMoor`). This - /// will allow for more descriptive error messages. - Future> parseTables( - Iterable types, Element initializedBy) { - return Future.wait(types.map((type) { - if (!_tableTypeChecker.isAssignableFrom(type.element)) { - reportError(ErrorInDartCode( - severity: Severity.criticalError, - message: 'The type $type is not a moor table', - affectedElement: initializedBy, - )); - return null; - } else { - return _parseTable(type.element as ClassElement); - } - })).then((list) { - // only keep tables that were resolved successfully - return List.from(list.where((t) => t != null)); - }); - } - - List readDeclaredQueries(Map obj) { - return obj.entries.map((entry) { - final key = entry.key.toStringValue(); - final value = entry.key.toStringValue(); - - return DeclaredQuery(key, value); - }).toList(); - } -} - -class ParseMoorFile extends Step { - final String content; - final TypeMapper mapper = TypeMapper(); - /* late final */ InlineDartResolver inlineDartResolver; - - ParseMoorFile(Task task, FoundFile file, this.content) : super(task, file) { - inlineDartResolver = InlineDartResolver(this); - } - - Future parseFile() { - final parser = MoorParser(this); - return parser.parseAndAnalyze(); - } -} - -/// Analyzes the compiled queries found in a Dart file. -class AnalyzeDartStep extends Step { - AnalyzeDartStep(Task task, FoundFile file) : super(task, file); - - @override - final bool isParsing = false; - - void analyze() { - final parseResult = file.currentResult as ParsedDartFile; - - for (var accessor in parseResult.dbAccessors) { - final transitivelyAvailable = accessor.resolvedImports - .where((file) => file.type == FileType.moor) - .map((file) => file.currentResult as ParsedMoorFile) - .expand((file) => file.declaredTables); - final availableTables = - accessor.tables.followedBy(transitivelyAvailable).toList(); - accessor.allTables = availableTables; - - final parser = SqlParser(this, availableTables, accessor.queries); - parser.parse(); - - accessor.resolvedQueries = parser.foundQueries; - } - } -} diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart new file mode 100644 index 00000000..0d9b20da --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart @@ -0,0 +1,28 @@ +part of '../steps.dart'; + +/// Analyzes the compiled queries found in a Dart file. +class AnalyzeDartStep extends Step { + AnalyzeDartStep(Task task, FoundFile file) : super(task, file); + + @override + final bool isParsing = false; + + void analyze() { + final parseResult = file.currentResult as ParsedDartFile; + + for (var accessor in parseResult.dbAccessors) { + final transitivelyAvailable = accessor.resolvedImports + .where((file) => file.type == FileType.moor) + .map((file) => file.currentResult as ParsedMoorFile) + .expand((file) => file.declaredTables); + final availableTables = + accessor.tables.followedBy(transitivelyAvailable).toList(); + accessor.allTables = availableTables; + + final parser = SqlParser(this, availableTables, accessor.queries); + parser.parse(); + + accessor.resolvedQueries = parser.foundQueries; + } + } +} diff --git a/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart b/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart new file mode 100644 index 00000000..da7a6d3b --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart @@ -0,0 +1,111 @@ +part of '../steps.dart'; + +/// Extracts the following information from a Dart file: +/// - [SpecifiedTable]s, which are read from Dart classes extending `Table`. +/// - [SpecifiedDatabase]s, which are read from `@UseMoor`-annotated classes +/// - [SpecifiedDao]s, which are read from `@UseDao`-annotated classes. +/// +/// Notably, this step does not analyze defined queries. +class ParseDartStep extends Step { + static const _tableTypeChecker = const TypeChecker.fromRuntime(Table); + static const _useMoorChecker = const TypeChecker.fromRuntime(UseMoor); + static const _useDaoChecker = const TypeChecker.fromRuntime(UseDao); + + final LibraryElement library; + + MoorDartParser _parser; + MoorDartParser get parser => _parser; + + final Map _tables = {}; + + ParseDartStep(Task task, FoundFile file, this.library) : super(task, file) { + _parser = MoorDartParser(this); + } + + Future parse() async { + final reader = LibraryReader(library); + final databases = []; + final daos = []; + + for (var declaredClass in reader.classes) { + // check if the table inherits from the moor table class. The !isExactly + // check is here because we run this generator on moor itself and we get + // weird errors for the Table class itself. + if (_tableTypeChecker.isAssignableFrom(declaredClass) && + !_tableTypeChecker.isExactly(declaredClass)) { + await _parseTable(declaredClass); + } else { + for (var annotation in _useMoorChecker.annotationsOf(declaredClass)) { + final reader = ConstantReader(annotation); + databases.add(await parseDatabase(declaredClass, reader)); + } + + for (var annotation in _useDaoChecker.annotationsOf(declaredClass)) { + final reader = ConstantReader(annotation); + daos.add(await parseDao(declaredClass, reader)); + } + } + } + + return ParsedDartFile( + library: library, + declaredTables: _tables.values.toList(), + declaredDaos: daos, + declaredDatabases: databases, + ); + } + + Future _parseTable(ClassElement element) async { + if (!_tables.containsKey(element)) { + _tables[element] = await parser.parseTable(element); + } + return _tables[element]; + } + + /// Parses a [SpecifiedDatabase] from the [ClassElement] which was annotated + /// with `@UseMoor` and the [annotation] reader that reads the `@UseMoor` + /// annotation. + Future parseDatabase( + ClassElement element, ConstantReader annotation) { + return UseMoorParser(this).parseDatabase(element, annotation); + } + + /// Parses a [SpecifiedDao] from a class declaration that has a `UseDao` + /// [annotation]. + Future parseDao( + ClassElement element, ConstantReader annotation) { + return UseDaoParser(this).parseDao(element, annotation); + } + + /// Resolves a [SpecifiedTable] for the class of each [DartType] in [types]. + /// The [initializedBy] element should be the piece of code that caused the + /// parsing (e.g. the database class that is annotated with `@UseMoor`). This + /// will allow for more descriptive error messages. + Future> parseTables( + Iterable types, Element initializedBy) { + return Future.wait(types.map((type) { + if (!_tableTypeChecker.isAssignableFrom(type.element)) { + reportError(ErrorInDartCode( + severity: Severity.criticalError, + message: 'The type $type is not a moor table', + affectedElement: initializedBy, + )); + return null; + } else { + return _parseTable(type.element as ClassElement); + } + })).then((list) { + // only keep tables that were resolved successfully + return List.from(list.where((t) => t != null)); + }); + } + + List readDeclaredQueries(Map obj) { + return obj.entries.map((entry) { + final key = entry.key.toStringValue(); + final value = entry.value.toStringValue(); + + return DeclaredQuery(key, value); + }).toList(); + } +} diff --git a/moor_generator/lib/src/analyzer/runner/steps/parse_moor.dart b/moor_generator/lib/src/analyzer/runner/steps/parse_moor.dart new file mode 100644 index 00000000..58ff2032 --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/steps/parse_moor.dart @@ -0,0 +1,16 @@ +part of '../steps.dart'; + +class ParseMoorStep extends Step { + final String content; + final TypeMapper mapper = TypeMapper(); + /* late final */ InlineDartResolver inlineDartResolver; + + ParseMoorStep(Task task, FoundFile file, this.content) : super(task, file) { + inlineDartResolver = InlineDartResolver(this); + } + + Future parseFile() { + final parser = MoorParser(this); + return parser.parseAndAnalyze(); + } +} diff --git a/moor_generator/lib/src/analyzer/runner/task.dart b/moor_generator/lib/src/analyzer/runner/task.dart index 05064b88..7b016d0e 100644 --- a/moor_generator/lib/src/analyzer/runner/task.dart +++ b/moor_generator/lib/src/analyzer/runner/task.dart @@ -21,24 +21,33 @@ class Task { final MoorSession session; final BackendTask backend; - final Map _performedSteps = {}; + final Set _analyzedFiles = {}; final List _unhandled = []; Task(this.session, this.input, this.backend); /// Returns an iterable of [FoundFile]s that were analyzed by this task. - Iterable get analyzedFiles => _performedSteps.keys; + Iterable get analyzedFiles => _analyzedFiles; Future runTask() async { // step 1: parse all files included by the input _unhandled.clear(); _unhandled.add(input); while (_unhandled.isNotEmpty) { - await _parse(_unhandled.removeLast()); + final file = _unhandled.removeLast(); + final step = await _parse(file); + + // the step can be null when a file that has already been parsed or even + // analyzed is encountered (for instance because of an import) + if (step != null) { + file.errors.consume(step.errors); + } + + _analyzedFiles.add(file); } // step 2: resolve queries in the input - for (var file in _performedSteps.keys) { + for (var file in _analyzedFiles) { file.errors.clearNonParsingErrors(); await _analyze(file); } @@ -46,21 +55,20 @@ class Task { session.notifyTaskFinished(this); } - Future _parse(FoundFile file) async { - if (file.state != FileState.dirty) { + Future _parse(FoundFile file) async { + if (file.isParsed) { // already parsed, nothing to do :) - return; + return null; } - final resolvedImports = {}; - + Step createdStep; file.errors.clearAll(); + final resolvedImports = {}; switch (file.type) { case FileType.moor: final content = await backend.readMoor(file.uri); - final step = ParseMoorFile(this, file, content); - _performedSteps[file] = step; + final step = createdStep = ParseMoorStep(this, file, content); final parsed = await step.parseFile(); file.currentResult = parsed; @@ -76,13 +84,11 @@ class Task { } else { resolvedImports.add(found); } - file.errors.consume(step.errors); } break; case FileType.dart: final library = await backend.resolveDart(file.uri); - final step = ParseDartStep(this, file, library); - _performedSteps[file] = step; + final step = createdStep = ParseDartStep(this, file, library); final parsed = await step.parse(); file.currentResult = parsed; @@ -109,7 +115,6 @@ class Task { } accessor.resolvedImports = resolvedForAccessor; - file.errors.consume(step.errors); } break; default: @@ -119,35 +124,39 @@ class Task { file.state = FileState.parsed; session.fileGraph.setImports(file, resolvedImports.toList()); _notifyFilesNeedWork(resolvedImports); + return createdStep; } Future _analyze(FoundFile file) async { // skip if already analyzed. if (file.state == FileState.analyzed) return; + Step step; + switch (file.type) { case FileType.dart: - final step = AnalyzeDartStep(this, file)..analyze(); - file.errors.consume(step.errors); + step = AnalyzeDartStep(this, file)..analyze(); break; default: break; } file.state = FileState.analyzed; + if (step != null) { + file.errors.consume(step.errors); + } } void _notifyFilesNeedWork(Iterable files) { for (var file in files) { - if (!_performedSteps.containsKey(file) && !_unhandled.contains(file)) { + if (!_analyzedFiles.contains(file) && !_unhandled.contains(file)) { _unhandled.add(file); } } } void printErrors() { - final foundErrors = - _performedSteps.values.expand((step) => step.errors.errors); + final foundErrors = _analyzedFiles.expand((file) => file.errors.errors); if (foundErrors.isNotEmpty) { final log = backend.log; diff --git a/moor_generator/lib/src/writer/database_writer.dart b/moor_generator/lib/src/writer/database_writer.dart index b2a920e1..583f8bd8 100644 --- a/moor_generator/lib/src/writer/database_writer.dart +++ b/moor_generator/lib/src/writer/database_writer.dart @@ -27,7 +27,7 @@ class DatabaseWriter { final tableGetters = []; - for (var table in db.tables) { + for (var table in db.allTables) { tableGetters.add(table.tableFieldName); final tableClassName = table.tableInfoName; diff --git a/moor_generator/test/parser/moor/moor_parser_test.dart b/moor_generator/test/parser/moor/moor_parser_test.dart index e4db549d..258678e5 100644 --- a/moor_generator/test/parser/moor/moor_parser_test.dart +++ b/moor_generator/test/parser/moor/moor_parser_test.dart @@ -10,7 +10,7 @@ CREATE TABLE users( '''; test('extracts table structure from .moor files', () async { - final parseStep = ParseMoorFile(null, null, content); + final parseStep = ParseMoorStep(null, null, content); final result = await parseStep.parseFile(); expect(parseStep.errors.errors, isEmpty); From 6a0716daaf8759c5ee7467e5bd0070c16fd6e5ad Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 12 Sep 2019 19:51:15 +0200 Subject: [PATCH 070/117] Handle import statements and declared queries in .moor --- .../lib/src/analyzer/moor/parser.dart | 20 +++++++---- .../lib/src/analyzer/runner/results.dart | 15 +++++++-- .../src/analyzer/runner/steps/parse_dart.dart | 2 +- .../lib/src/analyzer/runner/task.dart | 5 ++- .../src/analyzer/sql_queries/sql_parser.dart | 22 ++++++++----- moor_generator/lib/src/model/sql_query.dart | 33 +++++++++++++++++-- .../test/parser/moor/moor_parser_test.dart | 9 +++-- sqlparser/lib/src/engine/sql_engine.dart | 21 ++++++++++-- 8 files changed, 103 insertions(+), 24 deletions(-) diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index 17ca7737..e4b41563 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -2,6 +2,7 @@ import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/analyzer/moor/create_table_reader.dart'; import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; import 'package:sqlparser/sqlparser.dart'; class MoorParser { @@ -15,18 +16,18 @@ class MoorParser { final parsedFile = result.rootNode as MoorFile; final createdReaders = []; + final queryDeclarations = []; + final importStatements = []; for (var parsedStmt in parsedFile.statements) { if (parsedStmt is ImportStatement) { final importStmt = parsedStmt; step.inlineDartResolver.importStatements.add(importStmt.importedFile); + importStatements.add(importStmt); } else if (parsedStmt is CreateTableStatement) { createdReaders.add(CreateTableReader(parsedStmt)); - } else { - step.reportError(ErrorInMoorFile( - span: parsedStmt.span, - message: 'At the moment, only CREATE TABLE statements are supported' - 'in .moor files')); + } else if (parsedStmt is DeclaredStatement) { + queryDeclarations.add(DeclaredMoorQuery.fromStatement(parsedStmt)); } } @@ -40,6 +41,13 @@ class MoorParser { final createdTables = createdReaders.map((r) => r.extractTable(step.mapper)).toList(); - return Future.value(ParsedMoorFile(result, declaredTables: createdTables)); + return Future.value( + ParsedMoorFile( + result, + declaredTables: createdTables, + queries: queryDeclarations, + imports: importStatements, + ), + ); } } diff --git a/moor_generator/lib/src/analyzer/runner/results.dart b/moor_generator/lib/src/analyzer/runner/results.dart index cb3adb37..c9a403ca 100644 --- a/moor_generator/lib/src/analyzer/runner/results.dart +++ b/moor_generator/lib/src/analyzer/runner/results.dart @@ -1,7 +1,9 @@ import 'package:meta/meta.dart'; import 'package:analyzer/dart/element/element.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/model/specified_db_classes.dart'; import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; import 'package:sqlparser/sqlparser.dart'; abstract class FileResult {} @@ -26,7 +28,16 @@ class ParsedDartFile extends FileResult { class ParsedMoorFile extends FileResult { final ParseResult parseResult; MoorFile get parsedFile => parseResult.rootNode as MoorFile; - final List declaredTables; - ParsedMoorFile(this.parseResult, {this.declaredTables = const []}); + final List imports; + final List declaredTables; + final List queries; + + List resolvedQueries; + Map resolvedImports; + + ParsedMoorFile(this.parseResult, + {this.declaredTables = const [], + this.queries = const [], + this.imports = const []}); } diff --git a/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart b/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart index da7a6d3b..86d3fcb4 100644 --- a/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart +++ b/moor_generator/lib/src/analyzer/runner/steps/parse_dart.dart @@ -105,7 +105,7 @@ class ParseDartStep extends Step { final key = entry.key.toStringValue(); final value = entry.value.toStringValue(); - return DeclaredQuery(key, value); + return DeclaredDartQuery(key, value); }).toList(); } } diff --git a/moor_generator/lib/src/analyzer/runner/task.dart b/moor_generator/lib/src/analyzer/runner/task.dart index 7b016d0e..3748e83a 100644 --- a/moor_generator/lib/src/analyzer/runner/task.dart +++ b/moor_generator/lib/src/analyzer/runner/task.dart @@ -4,6 +4,7 @@ import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/backends/backend.dart'; import 'package:moor_generator/src/model/specified_db_classes.dart'; +import 'package:sqlparser/sqlparser.dart'; /// A task is used to fully parse and analyze files based on an input file. To /// analyze that file, all transitive imports will have to be analyzed as well. @@ -73,7 +74,8 @@ class Task { final parsed = await step.parseFile(); file.currentResult = parsed; - for (var import in parsed.parsedFile.imports) { + parsed.resolvedImports = {}; + for (var import in parsed.imports) { final found = session.resolve(file, import.importedFile); if (!await backend.exists(found.uri)) { step.reportError(ErrorInMoorFile( @@ -83,6 +85,7 @@ class Task { )); } else { resolvedImports.add(found); + parsed.resolvedImports[import] = found; } } break; diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index b989aaf0..213ee26c 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -29,16 +29,22 @@ class SqlParser { for (var query in definedQueries) { final name = query.name; - final sql = query.sql; AnalysisContext context; - try { - context = _engine.analyze(sql); - } catch (e, s) { - step.reportError(MoorError( - severity: Severity.criticalError, - message: 'Error while trying to parse $name: $e, $s')); - return; + + if (query is DeclaredDartQuery) { + final sql = query.sql; + + try { + context = _engine.analyze(sql); + } catch (e, s) { + step.reportError(MoorError( + severity: Severity.criticalError, + message: 'Error while trying to parse $name: $e, $s')); + return; + } + } else if (query is DeclaredMoorQuery) { + context = _engine.analyzeNode(query.query); } for (var error in context.errors) { diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 8d596ace..11f9c699 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -7,11 +7,40 @@ import 'package:sqlparser/sqlparser.dart'; final _illegalChars = RegExp(r'[^0-9a-zA-Z_]'); final _leadingDigits = RegExp(r'^\d*'); -class DeclaredQuery { +/// Represents the declaration of a compile-time query that will be analyzed +/// by moor_generator. +/// +/// The subclasses [DeclaredDartQuery] and [DeclaredMoorQuery] contain +/// information about the declared statement, only the name is common for both +/// declaration methods. +/// In the `analyze` step, a [DeclaredQuery] is turned into a resolved +/// [SqlQuery], which contains information about the affected tables and what +/// columns are returned. +abstract class DeclaredQuery { final String name; + + DeclaredQuery(this.name); +} + +/// A [DeclaredQuery] parsed from a Dart file by reading a constant annotation. +class DeclaredDartQuery extends DeclaredQuery { final String sql; - DeclaredQuery(this.name, this.sql); + DeclaredDartQuery(String name, this.sql) : super(name); +} + +/// A [DeclaredQuery] read from a `.moor` file, where the AST is already +/// available. +class DeclaredMoorQuery extends DeclaredQuery { + final AstNode query; + + DeclaredMoorQuery(String name, this.query) : super(name); + + factory DeclaredMoorQuery.fromStatement(DeclaredStatement stmt) { + final name = stmt.name; + final query = stmt.statement; + return DeclaredMoorQuery(name, query); + } } abstract class SqlQuery { diff --git a/moor_generator/test/parser/moor/moor_parser_test.dart b/moor_generator/test/parser/moor/moor_parser_test.dart index 258678e5..70f214ba 100644 --- a/moor_generator/test/parser/moor/moor_parser_test.dart +++ b/moor_generator/test/parser/moor/moor_parser_test.dart @@ -3,20 +3,25 @@ import 'package:test_api/test_api.dart'; void main() { final content = ''' +import 'package:my_package/some_file.dart'; +import 'relative_file.moor'; + CREATE TABLE users( id INT NOT NULL PRIMARY KEY AUTOINCREMENT, name VARCHAR NOT NULL CHECK(LENGTH(name) BETWEEN 5 AND 30) ); + +usersWithLongName: SELECT * FROM users WHERE LENGTH(name) > 25 '''; - test('extracts table structure from .moor files', () async { + test('parses standalone .moor files', () async { final parseStep = ParseMoorStep(null, null, content); final result = await parseStep.parseFile(); expect(parseStep.errors.errors, isEmpty); final table = result.declaredTables.single; - expect(table.sqlName, 'users'); + expect(table.columns.map((c) => c.name.name), ['id', 'name']); }); } diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 139f7f4f..0d0af363 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -91,6 +91,25 @@ class SqlEngine { final node = result.rootNode; final context = AnalysisContext(node, result.sql); + _analyzeContext(context); + + return context; + } + + /// Analyzes the given [node], which should be a [CrudStatement]. + /// The [AnalysisContext] enhances the AST by reporting type hints and errors. + /// + /// The analyzer needs to know all the available tables to resolve references + /// and result columns, so all known tables should be registered using + /// [registerTable] before calling this method. + AnalysisContext analyzeNode(AstNode node) { + final context = AnalysisContext(node, node.span.context); + _analyzeContext(context); + return context; + } + + void _analyzeContext(AnalysisContext context) { + final node = context.root; final scope = _constructRootScope(); try { @@ -106,8 +125,6 @@ class SqlEngine { // todo should we do now? AFAIK, everything that causes an exception // is added as an error contained in the context. } - - return context; } } From 156ef1ceb5b080c44d5ea894bca484f7c1cd7f3c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Thu, 12 Sep 2019 21:08:30 +0200 Subject: [PATCH 071/117] Support queries declared in .moor files --- .../lib/src/analyzer/runner/results.dart | 18 +++++--- .../lib/src/analyzer/runner/steps.dart | 17 +++++++ .../analyzer/runner/steps/analyze_dart.dart | 27 +++++++----- .../analyzer/runner/steps/analyze_moor.dart | 19 ++++++++ .../lib/src/analyzer/runner/task.dart | 44 ++++++++++++++++++- .../src/analyzer/sql_queries/sql_parser.dart | 2 +- .../lib/src/model/specified_db_classes.dart | 3 ++ 7 files changed, 108 insertions(+), 22 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart diff --git a/moor_generator/lib/src/analyzer/runner/results.dart b/moor_generator/lib/src/analyzer/runner/results.dart index c9a403ca..ba64ef07 100644 --- a/moor_generator/lib/src/analyzer/runner/results.dart +++ b/moor_generator/lib/src/analyzer/runner/results.dart @@ -6,12 +6,15 @@ import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; import 'package:sqlparser/sqlparser.dart'; -abstract class FileResult {} +abstract class FileResult { + final List declaredTables; + + FileResult(this.declaredTables); +} class ParsedDartFile extends FileResult { final LibraryElement library; - final List declaredTables; final List declaredDaos; final List declaredDatabases; @@ -20,9 +23,10 @@ class ParsedDartFile extends FileResult { ParsedDartFile( {@required this.library, - this.declaredTables = const [], + List declaredTables = const [], this.declaredDaos = const [], - this.declaredDatabases = const []}); + this.declaredDatabases = const []}) + : super(declaredTables); } class ParsedMoorFile extends FileResult { @@ -30,14 +34,14 @@ class ParsedMoorFile extends FileResult { MoorFile get parsedFile => parseResult.rootNode as MoorFile; final List imports; - final List declaredTables; final List queries; List resolvedQueries; Map resolvedImports; ParsedMoorFile(this.parseResult, - {this.declaredTables = const [], + {List declaredTables = const [], this.queries = const [], - this.imports = const []}); + this.imports = const []}) + : super(declaredTables); } diff --git a/moor_generator/lib/src/analyzer/runner/steps.dart b/moor_generator/lib/src/analyzer/runner/steps.dart index 9ed58f15..9fc54fb9 100644 --- a/moor_generator/lib/src/analyzer/runner/steps.dart +++ b/moor_generator/lib/src/analyzer/runner/steps.dart @@ -17,6 +17,7 @@ import 'package:moor_generator/src/model/sql_query.dart'; import 'package:source_gen/source_gen.dart'; part 'steps/analyze_dart.dart'; +part 'steps/analyze_moor.dart'; part 'steps/parse_dart.dart'; part 'steps/parse_moor.dart'; @@ -35,3 +36,19 @@ abstract class Step { void reportError(MoorError error) => errors.report(error..wasDuringParsing = isParsing); } + +abstract class AnalyzingStep extends Step { + AnalyzingStep(Task task, FoundFile file) : super(task, file); + + @override + final bool isParsing = false; + + List _transitiveImports(Iterable directImports) { + return task.crawlImports(directImports).toList(); + } + + Iterable _availableTables(List imports) { + return imports.expand( + (file) => file.currentResult?.declaredTables ?? const Iterable.empty()); + } +} diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart index 0d9b20da..8ee8fd46 100644 --- a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart @@ -1,28 +1,31 @@ part of '../steps.dart'; /// Analyzes the compiled queries found in a Dart file. -class AnalyzeDartStep extends Step { +class AnalyzeDartStep extends AnalyzingStep { AnalyzeDartStep(Task task, FoundFile file) : super(task, file); - @override - final bool isParsing = false; - void analyze() { final parseResult = file.currentResult as ParsedDartFile; for (var accessor in parseResult.dbAccessors) { - final transitivelyAvailable = accessor.resolvedImports - .where((file) => file.type == FileType.moor) - .map((file) => file.currentResult as ParsedMoorFile) - .expand((file) => file.declaredTables); - final availableTables = - accessor.tables.followedBy(transitivelyAvailable).toList(); - accessor.allTables = availableTables; + final transitiveImports = _transitiveImports(accessor.resolvedImports); + + final availableTables = _availableTables(transitiveImports) + .followedBy(accessor.tables) + .toList(); + + final availableQueries = transitiveImports + .map((f) => f.currentResult) + .whereType() + .expand((f) => f.resolvedQueries); final parser = SqlParser(this, availableTables, accessor.queries); parser.parse(); - accessor.resolvedQueries = parser.foundQueries; + accessor.allTables = availableTables; + + accessor.resolvedQueries = + availableQueries.followedBy(parser.foundQueries).toList(); } } } diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart new file mode 100644 index 00000000..8ec99458 --- /dev/null +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart @@ -0,0 +1,19 @@ +part of '../steps.dart'; + +class AnalyzeMoorStep extends AnalyzingStep { + AnalyzeMoorStep(Task task, FoundFile file) : super(task, file); + + void analyze() { + final parseResult = file.currentResult as ParsedMoorFile; + + final transitiveImports = + task.crawlImports(parseResult.resolvedImports.values).toList(); + + final availableTables = _availableTables(transitiveImports) + .followedBy(parseResult.declaredTables) + .toList(); + + final parser = SqlParser(this, availableTables, parseResult.queries); + parseResult.resolvedQueries = parser.foundQueries; + } +} diff --git a/moor_generator/lib/src/analyzer/runner/task.dart b/moor_generator/lib/src/analyzer/runner/task.dart index 3748e83a..c1e0f222 100644 --- a/moor_generator/lib/src/analyzer/runner/task.dart +++ b/moor_generator/lib/src/analyzer/runner/task.dart @@ -1,5 +1,6 @@ import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/backends/backend.dart'; @@ -47,8 +48,14 @@ class Task { _analyzedFiles.add(file); } - // step 2: resolve queries in the input - for (var file in _analyzedFiles) { + // step 2: resolve queries in the input. + // todo we force that moor files are analyzed first because they contain + // resolved queries which are copied into database accessors. Can we find + // a way to remove this special-handling? + final moorFiles = _analyzedFiles.where((f) => f.type == FileType.moor); + final otherFiles = _analyzedFiles.where((f) => f.type != FileType.moor); + + for (var file in moorFiles.followedBy(otherFiles)) { file.errors.clearNonParsingErrors(); await _analyze(file); } @@ -130,6 +137,36 @@ class Task { return createdStep; } + /// Crawls through all (transitive) imports of the provided [roots]. Each + /// [FoundFile] in the iterable provides queries and tables that are available + /// to the entity that imports them. + /// + /// This is different to [FileGraph.crawl] because imports are not accurate on + /// Dart files: Two accessors in a single Dart file could reference different + /// imports, but the [FileGraph] would only know about the union. + Iterable crawlImports(Iterable roots) sync* { + final found = {}; + final unhandled = roots.toList(); + + while (unhandled.isNotEmpty) { + final available = unhandled.removeLast(); + found.add(available); + yield available; + + var importsFromHere = const Iterable.empty(); + if (available.type == FileType.moor) { + importsFromHere = + (available.currentResult as ParsedMoorFile).resolvedImports.values; + } + + for (var next in importsFromHere) { + if (!found.contains(next) && !unhandled.contains(next)) { + unhandled.add(next); + } + } + } + } + Future _analyze(FoundFile file) async { // skip if already analyzed. if (file.state == FileState.analyzed) return; @@ -140,6 +177,9 @@ class Task { case FileType.dart: step = AnalyzeDartStep(this, file)..analyze(); break; + case FileType.moor: + step = AnalyzeMoorStep(this, file)..analyze(); + break; default: break; } diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index 213ee26c..6bd97383 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -9,7 +9,7 @@ import 'package:sqlparser/sqlparser.dart' hide ResultColumn; class SqlParser { final List tables; - final AnalyzeDartStep step; + final Step step; final List definedQueries; final TypeMapper _mapper = TypeMapper(); diff --git a/moor_generator/lib/src/model/specified_db_classes.dart b/moor_generator/lib/src/model/specified_db_classes.dart index a581c9bf..d07c1d70 100644 --- a/moor_generator/lib/src/model/specified_db_classes.dart +++ b/moor_generator/lib/src/model/specified_db_classes.dart @@ -12,6 +12,9 @@ class SpecifiedDbAccessor { final List queries; List resolvedImports = []; + + /// Resolved queries. This includes queries that weren't declared on this + /// class but imported via an `includes` directive. List resolvedQueries = const []; /// All tables available to this class. This includes the [tables] and all From e50e7b3f2176890309f5ba9af03bb204440eaa02 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 20:01:09 +0200 Subject: [PATCH 072/117] Write integration test for tables feature --- .../lib/src/analyzer/runner/steps.dart | 5 +- .../analyzer/runner/steps/analyze_moor.dart | 3 +- .../test/analyzer/integration_test.dart | 101 ++++++++++++++++++ moor_generator/test/utils/test_backend.dart | 2 +- 4 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 moor_generator/test/analyzer/integration_test.dart diff --git a/moor_generator/lib/src/analyzer/runner/steps.dart b/moor_generator/lib/src/analyzer/runner/steps.dart index 9fc54fb9..4f9ca856 100644 --- a/moor_generator/lib/src/analyzer/runner/steps.dart +++ b/moor_generator/lib/src/analyzer/runner/steps.dart @@ -44,7 +44,10 @@ abstract class AnalyzingStep extends Step { final bool isParsing = false; List _transitiveImports(Iterable directImports) { - return task.crawlImports(directImports).toList(); + return task + .crawlImports(directImports) + .where((import) => import != file) + .toList(); } Iterable _availableTables(List imports) { diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart index 8ec99458..95b9b783 100644 --- a/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart @@ -13,7 +13,8 @@ class AnalyzeMoorStep extends AnalyzingStep { .followedBy(parseResult.declaredTables) .toList(); - final parser = SqlParser(this, availableTables, parseResult.queries); + final parser = SqlParser(this, availableTables, parseResult.queries) + ..parse(); parseResult.resolvedQueries = parser.foundQueries; } } diff --git a/moor_generator/test/analyzer/integration_test.dart b/moor_generator/test/analyzer/integration_test.dart new file mode 100644 index 00000000..f5dd3038 --- /dev/null +++ b/moor_generator/test/analyzer/integration_test.dart @@ -0,0 +1,101 @@ +import 'package:build/build.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/analyzer/runner/task.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:moor_generator/src/model/specified_column.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:test/test.dart'; + +import '../utils/test_backend.dart'; + +void main() { + TestBackend backend; + MoorSession session; + + setUpAll(() { + backend = TestBackend( + { + AssetId.parse('test_lib|lib/database.dart'): r''' +import 'package:moor/moor.dart'; + +import 'another.dart'; // so that the resolver picks it up + +@DataClassName('UsesLanguage') +class UsedLanguages extends Table { + IntColumn get language => integer()(); + IntColumn get library => integer()(); + + @override + Set get primaryKey => {language, library}; +} + +@UseMoor( + tables: [UsedLanguages], + include: {'package:test_lib/tables.moor'}, + queries: { + 'transitiveImportTest': 'SELECT * FROM programming_languages', + }, +) +class Database {} + + ''', + AssetId.parse('test_lib|lib/tables.moor'): r''' +import 'another.dart'; + +CREATE TABLE libraries ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL +); + +findLibraries: SELECT * FROM libraries WHERE name LIKE ?; + ''', + AssetId.parse('test_lib|lib/another.dart'): r''' +import 'package:moor/moor.dart'; + +class ProgrammingLanguages extends Table { + IntColumn get id => integer().autoIncrement()(); + TextColumn get name => text()(); + IntColumn get popularity => integer().named('ieee_index').nullable()(); +} + ''', + }, + ); + session = backend.session; + }); + + tearDownAll(() { + backend.finish(); + }); + + Task task; + + setUp(() async { + final backendTask = + backend.startTask(Uri.parse('package:test_lib/database.dart')); + task = session.startTask(backendTask); + await task.runTask(); + }); + + test('resolves tables and queries', () { + final file = + session.registerFile(Uri.parse('package:test_lib/database.dart')); + + expect(file.state, FileState.analyzed); + expect(file.errors.errors, isEmpty); + + final result = file.currentResult as ParsedDartFile; + final database = result.declaredDatabases.single; + + expect(database.allTables.map((t) => t.sqlName), + containsAll(['used_languages', 'libraries', 'programming_languages'])); + + final importQuery = database.resolvedQueries + .singleWhere((q) => q.name == 'transitiveImportTest') as SqlSelectQuery; + expect(importQuery.resultClassName, 'ProgrammingLanguage'); + + final librariesQuery = database.resolvedQueries + .singleWhere((q) => q.name == 'findLibraries') as SqlSelectQuery; + expect(librariesQuery.variables.single.type, ColumnType.text); + }); +} diff --git a/moor_generator/test/utils/test_backend.dart b/moor_generator/test/utils/test_backend.dart index dfc8e41d..62de782c 100644 --- a/moor_generator/test/utils/test_backend.dart +++ b/moor_generator/test/utils/test_backend.dart @@ -75,6 +75,6 @@ class _TestBackendTask extends BackendTask { @override Future exists(Uri uri) async { - return backend.fakeContent.containsKey(AssetId.parse(uri.toString())); + return backend.fakeContent.containsKey(AssetId.resolve(uri.toString())); } } From 8ff5a7625e44e67734ef9df9953be484c67aa78a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 20:15:55 +0200 Subject: [PATCH 073/117] Write another integration test with cyclic queries --- .../analyzer/cyclic_moor_dart_dependency.dart | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 moor_generator/test/analyzer/cyclic_moor_dart_dependency.dart diff --git a/moor_generator/test/analyzer/cyclic_moor_dart_dependency.dart b/moor_generator/test/analyzer/cyclic_moor_dart_dependency.dart new file mode 100644 index 00000000..efff2bb6 --- /dev/null +++ b/moor_generator/test/analyzer/cyclic_moor_dart_dependency.dart @@ -0,0 +1,57 @@ +import 'package:build/build.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/analyzer/session.dart'; +import 'package:test/test.dart'; + +import '../utils/test_backend.dart'; + +void main() { + TestBackend backend; + MoorSession session; + + setUpAll(() { + backend = TestBackend({ + AssetId.parse('test_lib|lib/entry.dart'): r''' +import 'package:moor/moor.dart'; + +class Foos extends Table { + IntColumn get id => integer().autoIncrement()(); +} + +@UseMoor(include: {'db.moor'}, tables: [Foos]) +class Database {} + ''', + AssetId.parse('test_lib|lib/db.moor'): r''' +import 'entry.dart'; + +CREATE TABLE bars ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT +); + ''', + }); + session = backend.session; + }); + + tearDownAll(() { + backend.finish(); + }); + + test('handles cyclic imports', () async { + final backendTask = + backend.startTask(Uri.parse('package:test_lib/entry.dart')); + final task = session.startTask(backendTask); + await task.runTask(); + + final file = session.registerFile(Uri.parse('package:test_lib/entry.dart')); + + expect(file.state, FileState.analyzed); + expect(file.errors.errors, isEmpty); + + final result = file.currentResult as ParsedDartFile; + final database = result.declaredDatabases.single; + + expect(database.allTables.map((t) => t.sqlName), + containsAll(['foos', 'bars'])); + }); +} From 3097bb0591152eb7a53995e3d4956d6c585df9fe Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 20:23:48 +0200 Subject: [PATCH 074/117] Don't write get/watch methods for queries declared in moor --- moor/test/data/tables/custom_tables.g.dart | 21 +++++++++++++++++++ moor/test/data/tables/tables.moor | 4 +++- .../src/analyzer/sql_queries/sql_parser.dart | 6 +++++- moor_generator/lib/src/model/sql_query.dart | 6 ++++++ .../lib/src/writer/queries/query_writer.dart | 20 +++++++++--------- .../test/analyzer/integration_test.dart | 2 ++ 6 files changed, 47 insertions(+), 12 deletions(-) diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index d460692b..69bb85c8 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -812,6 +812,27 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Config get config => _config ??= Config(this); Mytable _mytable; Mytable get mytable => _mytable ??= Mytable(this); + ConfigData _rowToConfigData(QueryRow row) { + return ConfigData( + configKey: row.readString('config_key'), + configValue: row.readString('config_value'), + ); + } + + Selectable readConfig( + String var1, + {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') + QueryEngine operateOn}) { + return (operateOn ?? this).customSelectQuery( + 'readConfig: SELECT * FROM config WHERE config_key = ?;', + variables: [ + Variable.withString(var1), + ], + readsFrom: { + config + }).map(_rowToConfigData); + } + Future writeConfig( String key, String value, diff --git a/moor/test/data/tables/tables.moor b/moor/test/data/tables/tables.moor index a36c1b09..089cdf78 100644 --- a/moor/test/data/tables/tables.moor +++ b/moor/test/data/tables/tables.moor @@ -23,4 +23,6 @@ create table config ( CREATE TABLE mytable ( someid INTEGER NOT NULL PRIMARY KEY, sometext TEXT -) \ No newline at end of file +); + +readConfig: SELECT * FROM config WHERE config_key = ?; \ No newline at end of file diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index 6bd97383..b2af3755 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -29,6 +29,7 @@ class SqlParser { for (var query in definedQueries) { final name = query.name; + var declaredInMoor = false; AnalysisContext context; @@ -45,6 +46,7 @@ class SqlParser { } } else if (query is DeclaredMoorQuery) { context = _engine.analyzeNode(query.query); + declaredInMoor = true; } for (var error in context.errors) { @@ -55,7 +57,9 @@ class SqlParser { } try { - foundQueries.add(QueryHandler(name, context, _mapper).handle()); + final query = QueryHandler(name, context, _mapper).handle() + ..declaredInMoorFile = declaredInMoor; + foundQueries.add(query); } catch (e, s) { log.warning('Error while generating APIs for $name', e, s); } diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 11f9c699..1d064f2f 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -48,6 +48,12 @@ abstract class SqlQuery { final AnalysisContext fromContext; List lints; + /// Whether this query was declared in a `.moor` file. + /// + /// For those kind of queries, we don't generate `get` and `watch` methods and + /// instead only generate a single method returning a selectable. + bool declaredInMoorFile = false; + String get sql => fromContext.sql; /// The variables that appear in the [sql] query. We support three kinds of diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index e6233fd4..0a43f914 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -60,8 +60,11 @@ class QueryWriter { void _writeSelect() { _writeMapping(); _writeSelectStatementCreator(); - _writeOneTimeReader(); - _writeStreamReader(); + + if (!query.declaredInMoorFile) { + _writeOneTimeReader(); + _writeStreamReader(); + } } String _nameOfMappingMethod() { @@ -69,7 +72,11 @@ class QueryWriter { } String _nameOfCreationMethod() { - return '${_select.name}Query'; + if (query.declaredInMoorFile) { + return query.name; + } else { + return '${query.name}Query'; + } } /// Writes a mapping method that turns a "QueryRow" into the desired custom @@ -128,13 +135,6 @@ class QueryWriter { _buffer.write(');\n}\n'); } - /* - Future> allTodos(String name, - {QueryEngine overrideEngine}) { - return _allTodosWithCategoryQuery(name, engine: overrideEngine).get(); - } - */ - void _writeOneTimeReader() { _buffer.write('Future> ${query.name}('); _writeParameters(); diff --git a/moor_generator/test/analyzer/integration_test.dart b/moor_generator/test/analyzer/integration_test.dart index f5dd3038..16b18936 100644 --- a/moor_generator/test/analyzer/integration_test.dart +++ b/moor_generator/test/analyzer/integration_test.dart @@ -93,9 +93,11 @@ class ProgrammingLanguages extends Table { final importQuery = database.resolvedQueries .singleWhere((q) => q.name == 'transitiveImportTest') as SqlSelectQuery; expect(importQuery.resultClassName, 'ProgrammingLanguage'); + expect(importQuery.declaredInMoorFile, isFalse); final librariesQuery = database.resolvedQueries .singleWhere((q) => q.name == 'findLibraries') as SqlSelectQuery; expect(librariesQuery.variables.single.type, ColumnType.text); + expect(librariesQuery.declaredInMoorFile, isTrue); }); } From c8ae99b52ecc5633277425bd930f90357ca1cef1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 21:04:15 +0200 Subject: [PATCH 075/117] Breaking changes for the upcoming version 1. Removes transaction parameter in callbacks, custom queries 2. Removes MigrationStrategy.onFinished --- .../tests/lib/database/database.dart | 4 +- .../tests/lib/suite/transactions.dart | 4 +- moor/CHANGELOG.md | 33 ++++++++++ moor/example/example.dart | 5 +- moor/example/example.g.dart | 10 +-- moor/example_web/lib/database/database.dart | 2 +- moor/lib/src/runtime/database.dart | 12 ++-- moor/lib/src/runtime/migration.dart | 13 +--- moor/test/data/tables/custom_tables.g.dart | 11 +--- moor/test/data/tables/todos.g.dart | 65 +++++-------------- moor/test/database_test.dart | 5 +- moor/test/transactions_test.dart | 20 +++--- .../example/lib/database/database.dart | 4 +- .../lib/src/writer/queries/query_writer.dart | 23 ++----- 14 files changed, 86 insertions(+), 125 deletions(-) diff --git a/extras/integration_tests/tests/lib/database/database.dart b/extras/integration_tests/tests/lib/database/database.dart index c4590641..089890fe 100644 --- a/extras/integration_tests/tests/lib/database/database.dart +++ b/extras/integration_tests/tests/lib/database/database.dart @@ -98,7 +98,7 @@ class Database extends _$Database { await m.createTable(friendships); } }, - beforeOpen: (_, details) async { + beforeOpen: (details) async { if (details.wasCreated) { await into(users) .insertAll([People.dash, People.duke, People.gopher]); @@ -108,7 +108,7 @@ class Database extends _$Database { } Future deleteUser(User user, {bool fail = false}) { - return transaction((_) async { + return transaction(() async { final id = user.id; await (delete(friendships) ..where((f) => or(f.firstUser.equals(id), f.secondUser.equals(id)))) diff --git a/extras/integration_tests/tests/lib/suite/transactions.dart b/extras/integration_tests/tests/lib/suite/transactions.dart index 9b0e41ad..05b6f3b9 100644 --- a/extras/integration_tests/tests/lib/suite/transactions.dart +++ b/extras/integration_tests/tests/lib/suite/transactions.dart @@ -9,7 +9,7 @@ void transactionTests(TestExecutor executor) { final db = Database(executor.createExecutor()); // ignore: invalid_use_of_protected_member, invalid_use_of_visible_for_testing_member - await db.transaction((_) async { + await db.transaction(() async { final florianId = await db.writeUser(People.florian); final dash = await db.getUserById(People.dashId); @@ -32,7 +32,7 @@ void transactionTests(TestExecutor executor) { try { // ignore: invalid_use_of_protected_member, invalid_use_of_visible_for_testing_member - await db.transaction((_) async { + await db.transaction(() async { final florianId = await db.writeUser(People.florian); final dash = await db.getUserById(People.dashId); diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 3bee3386..d0ab5e19 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,3 +1,36 @@ +## 2.0.0 +This is the first major update after the initial release and moor and we have a lot to cover. +... Finally, we also removed a variety of deprecated features. See the breaking changes +section to learn what components are affected and what alternatives are available. + +TODO: Properly describe these additions when they're finalized: + +- Queries and imports in `.moor` files +- Analyzer plugin for Dart Code +- `ffi` libraries + +### Breaking changes +- __THIS LIKELY AFFECTS YOUR APP:__ Removed the `transaction` parameter for callbacks + in transactions and `beforeOpen` callbacks. So, instead of writing + ```dart + transaction((t) async { + await t.update(table)...; + }); + ``` + simply write + ```dart + transaction(() async { + await update(table)...; + }); + ``` + Similarly, instead of using `onOpen: (db, details) async {...}`, use + `onOpen: (details) async {...}`. You don't have to worry about calling methods on + your database instead of a transaction objects. They will be delegated automatically. + + On a similar note, we also removed the `operateOn` parameter from compiled queries. + +- Removed `MigrationStrategy.onFinished`. Use `beforeOpen` instead. + ## 1.7.2 - Fixed a race condition that caused the database to be opened multiple times on slower devices. This problem was introduced in `1.7.0` and was causing problems during migrations. diff --git a/moor/example/example.dart b/moor/example/example.dart index 062a0f8e..84889677 100644 --- a/moor/example/example.dart +++ b/moor/example/example.dart @@ -59,10 +59,9 @@ class Database extends _$Database { @override MigrationStrategy get migration { return MigrationStrategy( - beforeOpen: (engine, details) async { + beforeOpen: (details) async { // populate data - await engine - .into(categories) + await into(categories) .insert(const CategoriesCompanion(description: Value('Sweets'))); }, ); diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index 68b79aec..68eec0dd 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -847,19 +847,15 @@ abstract class _$Database extends GeneratedDatabase { ); } - Selectable _totalWeightQuery( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable _totalWeightQuery() { return (operateOn ?? this).customSelectQuery( ' SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id\n ', variables: [], readsFrom: {recipes, ingredientInRecipes}).map(_rowToTotalWeightResult); } - Future> _totalWeight( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return _totalWeightQuery(operateOn: operateOn).get(); + Future> _totalWeight() { + return _totalWeightQuery().get(); } Stream> _watchTotalWeight() { diff --git a/moor/example_web/lib/database/database.dart b/moor/example_web/lib/database/database.dart index 69eda725..ba259f5c 100644 --- a/moor/example_web/lib/database/database.dart +++ b/moor/example_web/lib/database/database.dart @@ -28,7 +28,7 @@ class Database extends _$Database { MigrationStrategy get migration { return MigrationStrategy( onCreate: (m) async => await m.createAllTables(), - beforeOpen: (engine, details) async { + beforeOpen: (details) async { if (details.wasCreated) { // populate default data await createTodoEntry( diff --git a/moor/lib/src/runtime/database.dart b/moor/lib/src/runtime/database.dart index 0abeb098..3c246668 100644 --- a/moor/lib/src/runtime/database.dart +++ b/moor/lib/src/runtime/database.dart @@ -280,10 +280,10 @@ mixin QueryEngine on DatabaseConnectionUser { /// inside a transaction returns the parent transaction. @protected @visibleForTesting - Future transaction(Future Function(QueryEngine transaction) action) async { + Future transaction(Future Function() action) async { final resolved = _resolvedEngine; if (resolved is Transaction) { - return action(resolved); + return action(); } final executor = resolved.executor; @@ -294,7 +294,7 @@ mixin QueryEngine on DatabaseConnectionUser { return _runEngineZoned(transaction, () async { var success = false; try { - await action(transaction); + await action(); success = true; } catch (e) { await transactionExecutor.rollback(); @@ -376,13 +376,11 @@ abstract class GeneratedDatabase extends DatabaseConnectionUser Future beforeOpenCallback( QueryExecutor executor, OpeningDetails details) async { final migration = _resolvedMigration; - if (migration.onFinished != null) { - await migration.onFinished(); - } + if (migration.beforeOpen != null) { final engine = BeforeOpenEngine(this, executor); await _runEngineZoned(engine, () { - return migration.beforeOpen(engine, details); + return migration.beforeOpen(details); }); } } diff --git a/moor/lib/src/runtime/migration.dart b/moor/lib/src/runtime/migration.dart index d1ad8224..40ca77aa 100644 --- a/moor/lib/src/runtime/migration.dart +++ b/moor/lib/src/runtime/migration.dart @@ -16,8 +16,7 @@ typedef Future OnMigrationFinished(); /// Signature of a function that's called before a database is marked opened by /// moor, but after migrations took place. This is a suitable callback to to /// populate initial data or issue `PRAGMA` statements that you want to use. -typedef OnBeforeOpen = Future Function( - QueryEngine db, OpeningDetails details); +typedef OnBeforeOpen = Future Function(OpeningDetails details); Future _defaultOnCreate(Migrator m) => m.createAllTables(); Future _defaultOnUpdate(Migrator m, int from, int to) async => @@ -33,14 +32,6 @@ class MigrationStrategy { /// happened at a lower [GeneratedDatabase.schemaVersion]. final OnUpgrade onUpgrade; - /// Executes after the database is ready and all migrations ran, but before - /// any other queries will be executed, making this method suitable to - /// populate data. - @Deprecated( - 'This callback is broken and only exists for backwards compatibility. ' - 'Use beforeOpen instead') - final OnMigrationFinished onFinished; - /// Executes after the database is ready to be used (ie. it has been opened /// and all migrations ran), but before any other queries will be sent. This /// makes it a suitable place to populate data after the database has been @@ -51,8 +42,6 @@ class MigrationStrategy { this.onCreate = _defaultOnCreate, this.onUpgrade = _defaultOnUpdate, this.beforeOpen, - @Deprecated('This callback is broken. Use beforeOpen instead') - this.onFinished, }); } diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 69bb85c8..83b12f08 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -819,10 +819,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { ); } - Selectable readConfig( - String var1, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable readConfig(String var1) { return (operateOn ?? this).customSelectQuery( 'readConfig: SELECT * FROM config WHERE config_key = ?;', variables: [ @@ -833,11 +830,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { }).map(_rowToConfigData); } - Future writeConfig( - String key, - String value, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Future writeConfig(String key, String value) { return (operateOn ?? this).customInsert( 'REPLACE INTO config VALUES (:key, :value)', variables: [ diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index d0bcdf53..9d853efc 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1312,9 +1312,7 @@ abstract class _$TodoDb extends GeneratedDatabase { ); } - Selectable allTodosWithCategoryQuery( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable allTodosWithCategoryQuery() { return (operateOn ?? this).customSelectQuery( 'SELECT t.*, c.id as catId, c."desc" as catDesc FROM todos t INNER JOIN categories c ON c.id = t.category', variables: [], @@ -1324,20 +1322,15 @@ abstract class _$TodoDb extends GeneratedDatabase { }).map(_rowToAllTodosWithCategoryResult); } - Future> allTodosWithCategory( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return allTodosWithCategoryQuery(operateOn: operateOn).get(); + Future> allTodosWithCategory() { + return allTodosWithCategoryQuery().get(); } Stream> watchAllTodosWithCategory() { return allTodosWithCategoryQuery().watch(); } - Future deleteTodoById( - int var1, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Future deleteTodoById(int var1) { return (operateOn ?? this).customUpdate( 'DELETE FROM todos WHERE id = ?', variables: [ @@ -1357,12 +1350,7 @@ abstract class _$TodoDb extends GeneratedDatabase { ); } - Selectable withInQuery( - String var1, - String var2, - List var3, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable withInQuery(String var1, String var2, List var3) { var $highestIndex = 3; final expandedvar3 = $expandVar($highestIndex, var3.length); $highestIndex += var3.length; @@ -1378,13 +1366,8 @@ abstract class _$TodoDb extends GeneratedDatabase { }).map(_rowToTodoEntry); } - Future> withIn( - String var1, - String var2, - List var3, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return withInQuery(var1, var2, var3, operateOn: operateOn).get(); + Future> withIn(String var1, String var2, List var3) { + return withInQuery(var1, var2, var3).get(); } Stream> watchWithIn( @@ -1392,10 +1375,7 @@ abstract class _$TodoDb extends GeneratedDatabase { return withInQuery(var1, var2, var3).watch(); } - Selectable searchQuery( - int id, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable searchQuery(int id) { return (operateOn ?? this).customSelectQuery( 'SELECT * FROM todos WHERE CASE WHEN -1 = :id THEN 1 ELSE id = :id END', variables: [ @@ -1406,11 +1386,8 @@ abstract class _$TodoDb extends GeneratedDatabase { }).map(_rowToTodoEntry); } - Future> search( - int id, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return searchQuery(id, operateOn: operateOn).get(); + Future> search(int id) { + return searchQuery(id).get(); } Stream> watchSearch(int id) { @@ -1424,19 +1401,15 @@ abstract class _$TodoDb extends GeneratedDatabase { ); } - Selectable findCustomQuery( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable findCustomQuery() { return (operateOn ?? this).customSelectQuery( 'SELECT custom FROM table_without_p_k WHERE some_float < 10', variables: [], readsFrom: {tableWithoutPK}).map(_rowToFindCustomResult); } - Future> findCustom( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return findCustomQuery(operateOn: operateOn).get(); + Future> findCustom() { + return findCustomQuery().get(); } Stream> watchFindCustom() { @@ -1498,10 +1471,7 @@ mixin _$SomeDaoMixin on DatabaseAccessor { ); } - Selectable todosForUserQuery( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { + Selectable todosForUserQuery(int user) { return (operateOn ?? this).customSelectQuery( 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', variables: [ @@ -1514,11 +1484,8 @@ mixin _$SomeDaoMixin on DatabaseAccessor { }).map(_rowToTodoEntry); } - Future> todosForUser( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return todosForUserQuery(user, operateOn: operateOn).get(); + Future> todosForUser(int user) { + return todosForUserQuery(user).get(); } Stream> watchTodosForUser(int user) { diff --git a/moor/test/database_test.dart b/moor/test/database_test.dart index 0ebbdfb2..bdb256fc 100644 --- a/moor/test/database_test.dart +++ b/moor/test/database_test.dart @@ -15,8 +15,9 @@ class _FakeDb extends GeneratedDatabase { onUpgrade: (m, from, to) async { await m.issueCustomQuery('updated from $from to $to'); }, - beforeOpen: (db, details) async { - await db.customSelect( + beforeOpen: (details) async { + // this fake select query is verified via mocks + await customSelect( 'opened: ${details.versionBefore} to ${details.versionNow}'); }, ); diff --git a/moor/test/transactions_test.dart b/moor/test/transactions_test.dart index 2403cd5b..db15bb64 100644 --- a/moor/test/transactions_test.dart +++ b/moor/test/transactions_test.dart @@ -33,16 +33,16 @@ void main() { test("transactions don't allow creating streams", () { expect(() async { - await db.transaction((t) async { - t.select(db.users).watch(); + await db.transaction(() async { + db.select(db.users).watch(); }); }, throwsStateError); }); test('nested transactions use the outer transaction', () async { - await db.transaction((t) async { - await t.transaction((t2) async { - expect(t2, equals(t)); + await db.transaction(() async { + await db.transaction(() async { + // todo how can we test that these are really equal? }); // the outer callback has not completed yet, so shouldn't send @@ -55,7 +55,7 @@ void main() { test('code in callback uses transaction', () async { // notice how we call .select on the database, but it should be called on // transaction executor. - await db.transaction((_) async { + await db.transaction(() async { await db.select(db.users).get(); }); @@ -65,7 +65,7 @@ void main() { test('transactions rollback after errors', () async { final exception = Exception('oh no'); - final future = db.transaction((_) async { + final future = db.transaction(() async { throw exception; }); @@ -79,8 +79,8 @@ void main() { when(executor.transactions.runUpdate(any, any)) .thenAnswer((_) => Future.value(2)); - await db.transaction((t) async { - await t + await db.transaction(() async { + await db .update(db.users) .write(const UsersCompanion(name: Value('Updated name'))); @@ -95,7 +95,7 @@ void main() { }); test('the database is opened before starting a transaction', () async { - await db.transaction((t) async { + await db.transaction(() async { verify(executor.doWhenOpened(any)); }); }); diff --git a/moor_flutter/example/lib/database/database.dart b/moor_flutter/example/lib/database/database.dart index e48b539d..e8db92ea 100644 --- a/moor_flutter/example/lib/database/database.dart +++ b/moor_flutter/example/lib/database/database.dart @@ -63,7 +63,7 @@ class Database extends _$Database { await m.addColumn(todos, todos.targetDate); } }, - beforeOpen: (db, details) async { + beforeOpen: (details) async { if (details.wasCreated) { // create default categories and entries final workId = await into(categories) @@ -154,7 +154,7 @@ class Database extends _$Database { } Future deleteCategory(Category category) { - return transaction((t) async { + return transaction(() async { await _resetCategory(category.id); await delete(categories).delete(category); }); diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 0a43f914..03f079ca 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -9,9 +9,6 @@ import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; import 'package:sqlparser/sqlparser.dart'; -const queryEngineWarningDesc = - 'No longer needed with Moor 1.6 - see the changelog for details'; - const highestAssignedIndexVar = '\$highestIndex'; /// Writes the handling code for a query. The code emitted will be a method that @@ -157,9 +154,9 @@ class QueryWriter { } _buffer.write('Stream> $methodName('); - _writeParameters(dontOverrideEngine: true); + _writeParameters(); _buffer..write(') {\n')..write('return ${_nameOfCreationMethod()}('); - _writeUseParameters(dontUseEngine: true); + _writeUseParameters(); _buffer.write(').watch();\n}\n'); } @@ -187,7 +184,7 @@ class QueryWriter { _buffer..write(',);\n}\n'); } - void _writeParameters({bool dontOverrideEngine = false}) { + void _writeParameters() { final paramList = query.variables.map((v) { var dartType = dartTypeNames[v.type]; if (v.isArray) { @@ -197,25 +194,13 @@ class QueryWriter { }).join(', '); _buffer.write(paramList); - - // write named optional parameter to configure the query engine used to - // execute the statement, - if (!dontOverrideEngine) { - if (query.variables.isNotEmpty) _buffer.write(', '); - _buffer.write('{@Deprecated(${asDartLiteral(queryEngineWarningDesc)}) ' - 'QueryEngine operateOn}'); - } } /// Writes code that uses the parameters as declared by [_writeParameters], /// assuming that for each parameter, a variable with the same name exists /// in the current scope. - void _writeUseParameters({bool dontUseEngine = false}) { + void _writeUseParameters() { _buffer.write(query.variables.map((v) => v.dartParameterName).join(', ')); - if (!dontUseEngine) { - if (query.variables.isNotEmpty) _buffer.write(', '); - _buffer.write('operateOn: operateOn'); - } } // Some notes on parameters and generating query code: From 138652fdc4b72f76c09d7fc2af859cd2add8315e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 21:06:11 +0200 Subject: [PATCH 076/117] Remove DiffUtil and MoorAnimatedList --- moor/lib/diff_util.dart | 81 ------ .../lib/src/utils/android_diffutils_port.dart | 249 ------------------ moor/test/diff_util_test.dart | 28 -- moor_flutter/lib/moor_flutter.dart | 1 - moor_flutter/lib/src/animated_list.dart | 131 --------- 5 files changed, 490 deletions(-) delete mode 100644 moor/lib/diff_util.dart delete mode 100644 moor/lib/src/utils/android_diffutils_port.dart delete mode 100644 moor/test/diff_util_test.dart delete mode 100644 moor_flutter/lib/src/animated_list.dart diff --git a/moor/lib/diff_util.dart b/moor/lib/diff_util.dart deleted file mode 100644 index e1ab3119..00000000 --- a/moor/lib/diff_util.dart +++ /dev/null @@ -1,81 +0,0 @@ -/// A utility library to find an edit script that turns a list into another. -/// This is useful when displaying a updating stream of immutable lists in a -/// list that can be updated. -@Deprecated('Will be removed in moor 2.0') -library diff_util; - -import 'package:moor/src/utils/android_diffutils_port.dart' as impl; - -class EditAction { - /// The index of the first list on which this action should be applied. If - /// this action [isDelete], that index and the next [amount] indices should be - /// deleted. Otherwise, this index should be moved back by [amount] and - /// entries from the second list (starting at [indexFromOther]) should be - /// inserted into the gap. - final int index; - - /// The amount of entries affected by this action - final int amount; - - /// If this action [isInsert], this is the first index from the second list - /// from where the items should be taken from. - final int indexFromOther; - - /// Whether this action should delete entries from the first list - bool get isDelete => indexFromOther == null; - - /// Whether this action should insert entries into the first list - bool get isInsert => indexFromOther != null; - - EditAction(this.index, this.amount, this.indexFromOther); - - @override - String toString() { - if (isDelete) { - return 'EditAction: Delete $amount entries from the first list, starting ' - 'at index $index'; - } else { - return 'EditAction: Insert $amount entries into the first list, taking ' - 'them from the second list starting at $indexFromOther. The entries ' - 'should be written starting at index $index'; - } - } -} - -/// Finds the shortest edit script that turns list [a] into list [b]. -/// The implementation is ported from androids DiffUtil, which in turn -/// implements a variation of Eugene W. Myer's difference algorithm. The -/// algorithm is optimized for space and uses O(n) space to find the minimal -/// number of addition and removal operations between the two lists. It has -/// O(N + D^2) time performance, where D is the minimum amount of edits needed -/// to turn a into b. -List diff(List a, List b, - {bool Function(T a, T b) equals}) { - final defaultEquals = equals ?? (T a, T b) => a == b; - final snakes = impl.calculateDiff(impl.DiffInput(a, b, defaultEquals)); - final actions = []; - - var posOld = a.length; - var posNew = b.length; - for (var snake in snakes.reversed) { - final snakeSize = snake.size; - final endX = snake.x + snakeSize; - final endY = snake.y + snakeSize; - - if (endX < posOld) { - // starting (including) with index endX, delete posOld - endX chars from a - actions.add(EditAction(endX, posOld - endX, null)); - } - if (endY < posNew) { - // starting with index endX, insert posNex - endY characters into a. The - // characters should be taken from b, starting (including) at the index - // endY. The char that was at index endX should be pushed back. - actions.add(EditAction(endX, posNew - endY, endY)); - } - - posOld = snake.x; - posNew = snake.y; - } - - return actions; -} diff --git a/moor/lib/src/utils/android_diffutils_port.dart b/moor/lib/src/utils/android_diffutils_port.dart deleted file mode 100644 index 1284aa4e..00000000 --- a/moor/lib/src/utils/android_diffutils_port.dart +++ /dev/null @@ -1,249 +0,0 @@ -// ignore_for_file: cascade_invocations - -/* -This implementation is copied from the DiffUtil class of the android support -library, available at https://chromium.googlesource.com/android_tools/+/refs/heads/master/sdk/sources/android-25/android/support/v7/util/DiffUtil.java -It has the following license: - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -class Snake { - int x; - int y; - int size; - bool removal; - bool reverse; -} - -class Range { - int oldListStart, oldListEnd; - int newListStart, newListEnd; - - Range.nullFields(); - Range(this.oldListStart, this.oldListEnd, this.newListStart, this.newListEnd); -} - -class DiffInput { - final List from; - final List to; - final bool Function(T a, T b) equals; - - DiffInput(this.from, this.to, this.equals); - - bool areItemsTheSame(int fromPos, int toPos) { - return equals(from[fromPos], to[toPos]); - } -} - -@Deprecated('Will be removed in moor 2.0') -List calculateDiff(DiffInput input) { - final oldSize = input.from.length; - final newSize = input.to.length; - - final snakes = []; - final stack = []; - - stack.add(Range(0, oldSize, 0, newSize)); - - final max = oldSize + newSize + (oldSize - newSize).abs(); - - final forward = List(max * 2); - final backward = List(max * 2); - - final rangePool = []; - - while (stack.isNotEmpty) { - final range = stack.removeLast(); - final snake = _diffPartial(input, range.oldListStart, range.oldListEnd, - range.newListStart, range.newListEnd, forward, backward, max); - - if (snake != null) { - if (snake.size > 0) { - snakes.add(snake); - } - - // offset the snake to convert its coordinates from the Range's are to - // global - snake.x += range.oldListStart; - snake.y += range.newListStart; - - // add new ranges for left and right - final left = - rangePool.isEmpty ? Range.nullFields() : rangePool.removeLast(); - left.oldListStart = range.oldListStart; - left.newListStart = range.newListStart; - if (snake.reverse) { - left.oldListEnd = snake.x; - left.newListEnd = snake.y; - } else { - if (snake.removal) { - left.oldListEnd = snake.x - 1; - left.newListEnd = snake.y; - } else { - left.oldListEnd = snake.x; - left.newListEnd = snake.y - 1; - } - } - stack.add(left); - - final right = range; - if (snake.reverse) { - if (snake.removal) { - right.oldListStart = snake.x + snake.size + 1; - right.newListStart = snake.y + snake.size; - } else { - right.oldListStart = snake.x + snake.size; - right.newListStart = snake.y + snake.size + 1; - } - } else { - right.oldListStart = snake.x + snake.size; - right.newListStart = snake.y + snake.size; - } - stack.add(right); - } else { - rangePool.add(range); - } - } - - snakes.sort((a, b) { - final cmpX = a.x - b.x; - return cmpX == 0 ? a.y - b.y : cmpX; - }); - - // add root snake - final first = snakes.isEmpty ? null : snakes.first; - - if (first == null || first.x != 0 || first.y != 0) { - snakes.insert( - 0, - Snake() - ..x = 0 - ..y = 0 - ..removal = false - ..size = 0 - ..reverse = false); - } - - return snakes; -} - -Snake _diffPartial(DiffInput input, int startOld, int endOld, int startNew, - int endNew, List forward, List backward, int kOffset) { - final oldSize = endOld - startOld; - final newSize = endNew - startNew; - - if (endOld - startOld < 1 || endNew - startNew < 1) return null; - - final delta = oldSize - newSize; - final dLimit = (oldSize + newSize + 1) ~/ 2; - - forward.fillRange(kOffset - dLimit - 1, kOffset + dLimit + 1, 0); - backward.fillRange( - kOffset - dLimit - 1 + delta, kOffset + dLimit + 1 + delta, oldSize); - - final checkInFwd = delta.isOdd; - - for (var d = 0; d <= dLimit; d++) { - for (var k = -d; k <= d; k += 2) { - // find forward path - // we can reach k from k - 1 or k + 1. Check which one is further in the - // graph. - int x; - bool removal; - - if (k == -d || - k != d && forward[kOffset + k - 1] < forward[kOffset + k + 1]) { - x = forward[kOffset + k + 1]; - removal = false; - } else { - x = forward[kOffset + k - 1] + 1; - removal = true; - } - - // set y based on x - var y = x - k; - - // move diagonal as long as items match - while (x < oldSize && - y < newSize && - input.areItemsTheSame(startOld + x, startNew + y)) { - x++; - y++; - } - - forward[kOffset + k] = x; - - if (checkInFwd && k >= delta - d + 1 && k <= delta + d - 1) { - if (forward[kOffset + k] >= backward[kOffset + k]) { - final outSnake = Snake()..x = backward[kOffset + k]; - outSnake - ..y = outSnake.x - k - ..size = forward[kOffset + k] - backward[kOffset + k] - ..removal = removal - ..reverse = false; - - return outSnake; - } - } - } - - for (var k = -d; k <= d; k += 2) { - // find reverse path at k + delta, in reverse - final backwardK = k + delta; - int x; - bool removal; - - if (backwardK == d + delta || - backwardK != -d + delta && - backward[kOffset + backwardK - 1] < - backward[kOffset + backwardK + 1]) { - x = backward[kOffset + backwardK - 1]; - removal = false; - } else { - x = backward[kOffset + backwardK + 1] - 1; - removal = true; - } - - // set y based on x - var y = x - backwardK; - // move diagonal as long as items match - while (x > 0 && - y > 0 && - input.areItemsTheSame(startOld + x - 1, startNew + y - 1)) { - x--; - y--; - } - - backward[kOffset + backwardK] = x; - - if (!checkInFwd && k + delta >= -d && k + delta <= d) { - if (forward[kOffset + backwardK] >= backward[kOffset + backwardK]) { - final outSnake = Snake()..x = backward[kOffset + backwardK]; - outSnake - ..y = outSnake.x - backwardK - ..size = - forward[kOffset + backwardK] - backward[kOffset + backwardK] - ..removal = removal - ..reverse = true; - - return outSnake; - } - } - } - } - - throw StateError("Unexpected case: Please make sure the lists don't change " - 'during a diff'); -} diff --git a/moor/test/diff_util_test.dart b/moor/test/diff_util_test.dart deleted file mode 100644 index 3b838add..00000000 --- a/moor/test/diff_util_test.dart +++ /dev/null @@ -1,28 +0,0 @@ -import 'package:test_api/test_api.dart'; -import 'package:moor/diff_util.dart'; - -List applyEditScript(List a, List b, List actions) { - final copy = List.of(a); - - for (var action in actions) { - if (action.isDelete) { - final deleteStartIndex = action.index; - copy.removeRange(deleteStartIndex, deleteStartIndex + action.amount); - } else if (action.isInsert) { - final toAdd = b.getRange( - action.indexFromOther, action.indexFromOther + action.amount); - copy.insertAll(action.index, toAdd); - } - } - - return copy; -} - -void main() { - final a = ['a', 'b', 'c', 'a', 'b', 'b', 'a']; - final b = ['c', 'b', 'a', 'b', 'a', 'c']; - - test('diff matcher should produce a correct edit script', () { - expect(applyEditScript(a, b, diff(a, b)), b); - }); -} diff --git a/moor_flutter/lib/moor_flutter.dart b/moor_flutter/lib/moor_flutter.dart index 8ba481bf..c30d6e1e 100644 --- a/moor_flutter/lib/moor_flutter.dart +++ b/moor_flutter/lib/moor_flutter.dart @@ -12,7 +12,6 @@ import 'package:moor/moor.dart'; import 'package:moor/backends.dart'; import 'package:sqflite/sqflite.dart' as s; -export 'package:moor_flutter/src/animated_list.dart'; export 'package:moor/moor.dart'; /// Signature of a function that runs when a database doesn't exist on file. diff --git a/moor_flutter/lib/src/animated_list.dart b/moor_flutter/lib/src/animated_list.dart deleted file mode 100644 index d1507285..00000000 --- a/moor_flutter/lib/src/animated_list.dart +++ /dev/null @@ -1,131 +0,0 @@ -import 'dart:async'; - -import 'package:flutter/widgets.dart'; - -// ignore: deprecated_member_use -import 'package:moor/diff_util.dart'; - -typedef Widget ItemBuilder( - BuildContext context, T item, Animation anim); -typedef Widget RemovedItemBuilder( - BuildContext context, T item, Animation anim); - -/// An [AnimatedList] that shows the result of a moor query stream. -@Deprecated('Will be removed in moor 2.0. You could use the ' - 'animated_stream_list package as an alternative') -class MoorAnimatedList extends StatefulWidget { - final Stream> stream; - final ItemBuilder itemBuilder; - final RemovedItemBuilder removedItemBuilder; - - /// A function that decides whether two items are considered equal. By - /// default, `a == b` will be used. A customization is useful if the content - /// of items can change (e.g. when a title changes, you'd only want to change - /// one text and not let the item disappear to show up again). - final bool Function(T a, T b) equals; - - MoorAnimatedList( - {@required this.stream, - @required this.itemBuilder, - @required this.removedItemBuilder, - this.equals}); - - @override - _MoorAnimatedListState createState() { - return _MoorAnimatedListState(); - } -} - -class _MoorAnimatedListState extends State> { - List _lastSnapshot; - int _initialItemCount; - - StreamSubscription _subscription; - - final GlobalKey _key = GlobalKey(); - AnimatedListState get listState => _key.currentState; - - @override - void initState() { - _setupSubscription(); - super.initState(); - } - - void _receiveData(List data) { - if (listState == null) { - setState(() { - _lastSnapshot = data; - _initialItemCount = data.length; - }); - return; - } - - if (_lastSnapshot == null) { - // no diff possible. Initialize lists instead of diffing - _lastSnapshot = data; - for (var i = 0; i < data.length; i++) { - listState.insertItem(i); - } - } else { - final editScript = diff(_lastSnapshot, data, equals: widget.equals); - - for (var action in editScript) { - if (action.isDelete) { - // we need to delete action.amount items at index action.index - for (var i = 0; i < action.amount; i++) { - // i items have already been deleted, so + 1 for the index. Notice - // that we don't have to do this when calling removeItem on the - // animated list state, as it will reflect the operation immediately. - final itemHere = _lastSnapshot[action.index + i]; - listState.removeItem(action.index, (ctx, anim) { - return widget.removedItemBuilder(ctx, itemHere, anim); - }); - } - } else { - for (var i = 0; i < action.amount; i++) { - listState.insertItem(action.index + i); - } - } - } - - setState(() { - _lastSnapshot = data; - }); - } - } - - void _setupSubscription() { - _subscription = widget.stream.listen(_receiveData); - } - - @override - void didUpdateWidget(MoorAnimatedList oldWidget) { - _subscription?.cancel(); - _lastSnapshot = null; - _setupSubscription(); - - super.didUpdateWidget(oldWidget); - } - - @override - void dispose() { - _subscription?.cancel(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - if (_lastSnapshot == null) return const SizedBox(); - - return AnimatedList( - key: _key, - initialItemCount: _initialItemCount ?? 0, - itemBuilder: (ctx, index, anim) { - final item = _lastSnapshot[index]; - final child = widget.itemBuilder(ctx, item, anim); - - return child; - }, - ); - } -} From e9225cf7593b6ff15f0d07c1e1e2c65ba2519aaa Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 21:11:17 +0200 Subject: [PATCH 077/117] Always generate private watch methods --- .../docs/Advanced Features/builder_options.md | 9 +-------- moor/CHANGELOG.md | 4 ++++ .../lib/src/backends/build/options.dart | 17 +++-------------- .../lib/src/writer/queries/query_writer.dart | 5 ++--- 4 files changed, 10 insertions(+), 25 deletions(-) diff --git a/docs/content/en/docs/Advanced Features/builder_options.md b/docs/content/en/docs/Advanced Features/builder_options.md index c6897a1d..65b3a641 100644 --- a/docs/content/en/docs/Advanced Features/builder_options.md +++ b/docs/content/en/docs/Advanced Features/builder_options.md @@ -18,18 +18,11 @@ targets: builders: moor_generator: options: - generate_private_watch_methods: true + write_from_json_string_constructor: true ``` At the moment, moor supports these options: -* `generate_private_watch_methods`: boolean. There was a bug in the generator where - [compiled queries]({{}}) that start with - an underscore did generate a watch method that didn't start with an underscore - (see [#107](https://github.com/simolus3/moor/issues/107)). Fixing this would be - a breaking change, so the fix is opt-in by enabling this option. This flag is - available since 1.7 and will be removed in moor 2.0, where this flag will always - be enabled. * `write_from_json_string_constructor`: boolean. Adds a `.fromJsonString` factory constructor to generated data classes. By default, we only write a `.fromJson` constructor that takes a `Map`. \ No newline at end of file diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index d0ab5e19..312ca1fb 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -30,6 +30,10 @@ TODO: Properly describe these additions when they're finalized: On a similar note, we also removed the `operateOn` parameter from compiled queries. - Removed `MigrationStrategy.onFinished`. Use `beforeOpen` instead. +- Compiled sql queries starting with an underscore will now generate private match queries. + Previously, the query `_allUsers` would generate a `watchAllUsers` method, that has been + adopted to `_watchAllUsers`. The `generate_private_watch_methods` builder option, which + backported this fix to older versions, has thus been removed. ## 1.7.2 - Fixed a race condition that caused the database to be opened multiple times on slower devices. diff --git a/moor_generator/lib/src/backends/build/options.dart b/moor_generator/lib/src/backends/build/options.dart index c3bdf26c..581cc7f1 100644 --- a/moor_generator/lib/src/backends/build/options.dart +++ b/moor_generator/lib/src/backends/build/options.dart @@ -3,25 +3,14 @@ part of 'moor_builder.dart'; class MoorOptions { final bool generateFromJsonStringConstructor; - /// A bug in the generator generates public watch* methods, even if the query - /// name starts with an underscore. Fixing this would be a breaking change, so - /// we introduce a flag that will be the default behavior in the next breaking - /// moor version. - final bool fixPrivateWatchMethods; - - MoorOptions( - this.generateFromJsonStringConstructor, this.fixPrivateWatchMethods); + MoorOptions(this.generateFromJsonStringConstructor); factory MoorOptions.fromBuilder(Map config) { final writeFromString = config['write_from_json_string_constructor'] as bool ?? false; - final fixWatchMethods = - config['generate_private_watch_methods'] as bool ?? false; - return MoorOptions(writeFromString, fixWatchMethods); + return MoorOptions(writeFromString); } - const MoorOptions.defaults() - : generateFromJsonStringConstructor = false, - fixPrivateWatchMethods = false; + const MoorOptions.defaults() : generateFromJsonStringConstructor = false; } diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 03f079ca..472db00b 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -145,9 +145,8 @@ class QueryWriter { String methodName; // turning the query name into pascal case will remove underscores, add the - // "private" modifier back in if needed - if (scope.writer.options.fixPrivateWatchMethods && - query.name.startsWith('_')) { + // "private" modifier back in + if (query.name.startsWith('_')) { methodName = '_watch$upperQueryName'; } else { methodName = 'watch$upperQueryName'; From 456392333dc497836495ace7040cad68a607515c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 21:12:51 +0200 Subject: [PATCH 078/117] Remove InsertStatement.insertOrReplace --- moor/CHANGELOG.md | 1 + moor/lib/src/runtime/statements/insert.dart | 15 --------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 312ca1fb..26b352b5 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -34,6 +34,7 @@ TODO: Properly describe these additions when they're finalized: Previously, the query `_allUsers` would generate a `watchAllUsers` method, that has been adopted to `_watchAllUsers`. The `generate_private_watch_methods` builder option, which backported this fix to older versions, has thus been removed. +- Removed `InsertStatement.insertOrReplace`. Use `insert(data, orReplace: true)` instead. ## 1.7.2 - Fixed a race condition that caused the database to be opened multiple times on slower devices. diff --git a/moor/lib/src/runtime/statements/insert.dart b/moor/lib/src/runtime/statements/insert.dart index c7d837cc..edfcb933 100644 --- a/moor/lib/src/runtime/statements/insert.dart +++ b/moor/lib/src/runtime/statements/insert.dart @@ -4,8 +4,6 @@ import 'package:meta/meta.dart'; import 'package:moor/moor.dart'; import 'package:moor/src/runtime/components/component.dart'; -import 'update.dart'; - class InsertStatement { @protected final QueryEngine database; @@ -72,19 +70,6 @@ class InsertStatement { database.markTablesUpdated({table}); } - /// Updates the row with the same primary key in the database or creates one - /// if it doesn't exist. - /// - /// Behaves similar to [UpdateStatement.replace], meaning that all fields from - /// [entity] will be written to override rows with the same primary key, which - /// includes setting columns with null values back to null. - /// - /// However, if no such row exists, a new row will be written instead. - @Deprecated('Use insert with orReplace: true instead') - Future insertOrReplace(Insertable entity) async { - return await insert(entity, orReplace: true); - } - GenerationContext _createContext(Insertable entry, bool replace) { final map = table.entityToSql(entry.createCompanion(true)) ..removeWhere((_, value) => value == null); From f7ade2b7b6689c3b60ddee07d684d5fd9f5ba0de Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 21:25:14 +0200 Subject: [PATCH 079/117] Explain why we have more than one primaryKey override --- moor/lib/src/runtime/structure/table_info.dart | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/moor/lib/src/runtime/structure/table_info.dart b/moor/lib/src/runtime/structure/table_info.dart index ef8d7ed4..d0df7c2b 100644 --- a/moor/lib/src/runtime/structure/table_info.dart +++ b/moor/lib/src/runtime/structure/table_info.dart @@ -11,14 +11,13 @@ mixin TableInfo on Table { /// The primary key of this table. Can be null or empty if no custom primary /// key has been specified. + /// + /// Additional to the [Table.primaryKey] columns declared by an user, this + /// also contains auto-increment integers, which are primary key by default. Set get $primaryKey => null; - // The "primaryKey" is what users define on their table classes, the - // "$primaryKey" is what moor generates in the implementation table info - // classes. Having two of them is pretty pointless, we're going to remove - // the "$primaryKey$ getter in Moor 2.0. Until then, let's make sure they're - // consistent for classes from CREATE TABLE statements, where the info class - // and the table class is the same thing but primaryKey isn't overriden. + // ensure the primaryKey getter is consistent with $primarKey, which can + // contain additional columns. @override Set get primaryKey => $primaryKey; From 44a2319bba1f542dec496312205897c1d2ed589a Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 22:48:55 +0200 Subject: [PATCH 080/117] Improve variable tokenization for easier parsing logic --- sqlparser/lib/src/ast/ast.dart | 5 ++ sqlparser/lib/src/ast/clauses/limit.dart | 4 +- .../lib/src/ast/expressions/variables.dart | 11 +++-- sqlparser/lib/src/ast/moor/inline_dart.dart | 47 +++++++++++++++++++ .../lib/src/reader/parser/expressions.dart | 29 +++++------- .../lib/src/reader/tokenizer/scanner.dart | 36 +++++++++++++- sqlparser/lib/src/reader/tokenizer/token.dart | 34 ++++++++++++-- sqlparser/test/parser/expression_test.dart | 8 ++-- sqlparser/test/parser/utils.dart | 10 ++-- 9 files changed, 148 insertions(+), 36 deletions(-) create mode 100644 sqlparser/lib/src/ast/moor/inline_dart.dart diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index 003f41ac..a33dc882 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -23,6 +23,7 @@ part 'expressions/variables.dart'; part 'moor/declared_statement.dart'; part 'moor/import_statement.dart'; +part 'moor/inline_dart.dart'; part 'moor/moor_file.dart'; part 'schema/column_definition.dart'; @@ -191,6 +192,7 @@ abstract class AstVisitor { T visitMoorFile(MoorFile e); T visitMoorImportStatement(ImportStatement e); T visitMoorDeclaredStatement(DeclaredStatement e); + T visitInlineDartCode(InlineDart e); } /// Visitor that walks down the entire tree, visiting all children in order. @@ -312,6 +314,9 @@ class RecursiveVisitor extends AstVisitor { @override T visitMoorDeclaredStatement(DeclaredStatement e) => visitChildren(e); + @override + T visitInlineDartCode(InlineDart e) => visitChildren(e); + @protected T visitChildren(AstNode e) { for (var child in e.childNodes) { diff --git a/sqlparser/lib/src/ast/clauses/limit.dart b/sqlparser/lib/src/ast/clauses/limit.dart index 59b5a9c5..be3e32f9 100644 --- a/sqlparser/lib/src/ast/clauses/limit.dart +++ b/sqlparser/lib/src/ast/clauses/limit.dart @@ -1,6 +1,8 @@ part of '../ast.dart'; -class Limit extends AstNode { +abstract class LimitBase {} + +class Limit extends AstNode implements LimitBase { Expression count; Token offsetSeparator; // can either be OFFSET or just a comma Expression offset; diff --git a/sqlparser/lib/src/ast/expressions/variables.dart b/sqlparser/lib/src/ast/expressions/variables.dart index 28c8cb7f..08a428a8 100644 --- a/sqlparser/lib/src/ast/expressions/variables.dart +++ b/sqlparser/lib/src/ast/expressions/variables.dart @@ -6,10 +6,10 @@ mixin Variable on Expression { /// A "?" or "?123" variable placeholder class NumberedVariable extends Expression with Variable { - final Token questionMark; - final int explicitIndex; + final QuestionMarkVariableToken token; + int get explicitIndex => token.explicitIndex; - NumberedVariable(this.questionMark, this.explicitIndex); + NumberedVariable(this.token); @override T accept(AstVisitor visitor) { @@ -26,9 +26,10 @@ class NumberedVariable extends Expression with Variable { } class ColonNamedVariable extends Expression with Variable { - final String name; + final ColonVariableToken token; + String get name => token.name; - ColonNamedVariable(this.name); + ColonNamedVariable(this.token); @override T accept(AstVisitor visitor) { diff --git a/sqlparser/lib/src/ast/moor/inline_dart.dart b/sqlparser/lib/src/ast/moor/inline_dart.dart new file mode 100644 index 00000000..9efc6b30 --- /dev/null +++ b/sqlparser/lib/src/ast/moor/inline_dart.dart @@ -0,0 +1,47 @@ +part of '../ast.dart'; + +/// An inline Dart component that appears in a compiled sql query. Inline Dart +/// components can be bound with complex expressions at runtime by using moor's +/// Dart API. +/// +/// At the moment, we support 4 kind of inline components: +/// 1. expressions: Any expression can be used for moor: `SELECT * FROM table +/// = $expr`. Generated code will write this as an `Expression` class from +/// moor. +/// 2. limits +/// 3. A single order-by clause +/// 4. A list of order-by clauses +abstract class InlineDart extends AstNode { + final String name; + + DollarSignVariableToken token; + + InlineDart._(this.name); + + @override + final Iterable childNodes = const Iterable.empty(); + + @override + T accept(AstVisitor visitor) => visitor.visitInlineDartCode(this); + + bool _dartEquals(covariant InlineDart other); + + @override + bool contentEquals(InlineDart other) { + return other.name == name && other._dartEquals(other); + } +} + +class InlineDartExpression extends InlineDart implements Expression { + InlineDartExpression({@required String name}) : super._(name); + + @override + bool _dartEquals(InlineDartExpression other) => true; +} + +class InlineDartLimit extends InlineDart implements LimitBase { + InlineDartLimit({@required String name}) : super._(name); + + @override + bool _dartEquals(InlineDartLimit other) => true; +} diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index bdf6879c..48d0a624 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -313,25 +313,20 @@ mixin ExpressionParser on ParserBase { return Reference(columnName: first.identifier)..setSpan(first, first); } break; - case TokenType.questionMark: - final mark = token; - - if (_matchOne(TokenType.numberLiteral)) { - final number = _previous; - return NumberedVariable(mark, _parseNumber(number.lexeme).toInt()) - ..setSpan(mark, number); - } else { - return NumberedVariable(mark, null)..setSpan(mark, mark); + case TokenType.questionMarkVariable: + return NumberedVariable(token as QuestionMarkVariableToken) + ..setSpan(token, token); + case TokenType.colonVariable: + return ColonNamedVariable(token as ColonVariableToken) + ..setSpan(token, token); + case TokenType.dollarSignVariable: + if (enableMoorExtensions) { + final typedToken = token as DollarSignVariableToken; + return InlineDartExpression(name: typedToken.name) + ..token = typedToken + ..setSpan(token, token); } break; - case TokenType.colon: - final colon = token; - final identifier = _consumeIdentifier( - 'Expected an identifier for the named variable', - lenient: true); - - final content = identifier.identifier; - return ColonNamedVariable(':$content')..setSpan(colon, identifier); default: break; } diff --git a/sqlparser/lib/src/reader/tokenizer/scanner.dart b/sqlparser/lib/src/reader/tokenizer/scanner.dart index d388f572..71218ac2 100644 --- a/sqlparser/lib/src/reader/tokenizer/scanner.dart +++ b/sqlparser/lib/src/reader/tokenizer/scanner.dart @@ -112,10 +112,31 @@ class Scanner { break; case '?': - _addToken(TokenType.questionMark); + // if the next chars are numbers, this is an explicitly index variable + final buffer = StringBuffer(); + while (!_isAtEnd && isDigit(_peek())) { + buffer.write(_nextChar()); + } + + int explicitIndex; + if (buffer.isNotEmpty) { + explicitIndex = int.parse(buffer.toString()); + } + + tokens.add(QuestionMarkVariableToken(_currentSpan, explicitIndex)); break; case ':': - _addToken(TokenType.colon); + final name = _matchColumnName(); + if (name == null) { + _addToken(TokenType.colon); + } else { + tokens.add(ColonVariableToken(_currentSpan, ':$name')); + } + + break; + case r'$': + final name = _matchColumnName(); + tokens.add(ColonVariableToken(_currentSpan, name)); break; case ';': _addToken(TokenType.semicolon); @@ -330,6 +351,17 @@ class Scanner { } } + String _matchColumnName() { + if (_isAtEnd || !canStartColumnName(_peek())) return null; + + final buffer = StringBuffer()..write(_nextChar()); + while (!_isAtEnd && continuesColumnName(_peek())) { + buffer.write(_nextChar()); + } + + return buffer.toString(); + } + void _inlineDart() { // inline starts with a `, we just need to find the matching ` that // terminates this token. diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index 8f1a71fb..c47ec150 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -38,11 +38,12 @@ enum TokenType { exists, collate, - questionMark, + questionMarkVariable, colon, - // todo at and dollarSign are currently not used + colonVariable, + // todo at is not used at the moment at, - dollarSign, + dollarSignVariable, stringLiteral, numberLiteral, @@ -293,6 +294,33 @@ class IdentifierToken extends Token { : super(TokenType.identifier, span); } +abstract class VariableToken extends Token { + VariableToken(TokenType type, FileSpan span) : super(type, span); +} + +class QuestionMarkVariableToken extends Token { + /// The explicit index, if this variable was of the form `?123`. Otherwise + /// null. + final int explicitIndex; + + QuestionMarkVariableToken(FileSpan span, this.explicitIndex) + : super(TokenType.questionMarkVariable, span); +} + +class ColonVariableToken extends Token { + final String name; + + ColonVariableToken(FileSpan span, this.name) + : super(TokenType.colonVariable, span); +} + +class DollarSignVariableToken extends Token { + final String name; + + DollarSignVariableToken(FileSpan span, this.name) + : super(TokenType.dollarSignVariable, span); +} + /// Inline Dart appearing in a create table statement. Only parsed when the moor /// extensions are enabled. Dart code is wrapped in backticks. class InlineDartToken extends Token { diff --git a/sqlparser/test/parser/expression_test.dart b/sqlparser/test/parser/expression_test.dart index fae06cfc..f0508dc2 100644 --- a/sqlparser/test/parser/expression_test.dart +++ b/sqlparser/test/parser/expression_test.dart @@ -41,15 +41,15 @@ final Map _testCases = { '? * ?3 + ?2 == :test': BinaryExpression( BinaryExpression( BinaryExpression( - NumberedVariable(token(TokenType.questionMark), null), + NumberedVariable(QuestionMarkVariableToken(fakeSpan('?'), null)), token(TokenType.star), - NumberedVariable(token(TokenType.questionMark), 3), + NumberedVariable(QuestionMarkVariableToken(fakeSpan('?3'), 3)), ), token(TokenType.plus), - NumberedVariable(token(TokenType.questionMark), 2), + NumberedVariable(QuestionMarkVariableToken(fakeSpan('?2'), 2)), ), token(TokenType.doubleEqual), - ColonNamedVariable(':test'), + ColonNamedVariable(ColonVariableToken(fakeSpan(':test'), ':test')), ), 'CASE x WHEN a THEN b WHEN c THEN d ELSE e END': CaseExpression( base: Reference(columnName: 'x'), diff --git a/sqlparser/test/parser/utils.dart b/sqlparser/test/parser/utils.dart index e7b3b75f..b88573d4 100644 --- a/sqlparser/test/parser/utils.dart +++ b/sqlparser/test/parser/utils.dart @@ -11,13 +11,11 @@ Token token(TokenType type) { } InlineDartToken inlineDart(String dartCode) { - final fakeFile = SourceFile.fromString('`$dartCode`'); - return InlineDartToken(fakeFile.span(0)); + return InlineDartToken(fakeSpan('`$dartCode`')); } IdentifierToken identifier(String content) { - final fakeFile = SourceFile.fromString(content); - return IdentifierToken(false, fakeFile.span(0)); + return IdentifierToken(false, fakeSpan(content)); } void testStatement(String sql, AstNode expected) { @@ -25,6 +23,10 @@ void testStatement(String sql, AstNode expected) { enforceEqual(parsed, expected); } +FileSpan fakeSpan(String content) { + return SourceFile.fromString(content).span(0); +} + void testAll(Map testCases) { testCases.forEach((sql, expected) { test('with $sql', () { From 4c250c8f4064b2bd1301a63b3b05ce87656865ea Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 13 Sep 2019 23:08:29 +0200 Subject: [PATCH 081/117] Parse dart template expressions and limit clauses --- sqlparser/lib/src/ast/clauses/limit.dart | 2 +- sqlparser/lib/src/ast/statements/select.dart | 2 +- sqlparser/lib/src/reader/parser/crud.dart | 19 ++++++++--- .../lib/src/reader/tokenizer/scanner.dart | 2 +- .../test/analysis/variable_indexing_test.dart | 2 +- sqlparser/test/parser/inline_dart_test.dart | 34 +++++++++++++++++++ sqlparser/test/parser/utils.dart | 4 +-- 7 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 sqlparser/test/parser/inline_dart_test.dart diff --git a/sqlparser/lib/src/ast/clauses/limit.dart b/sqlparser/lib/src/ast/clauses/limit.dart index be3e32f9..08e56f4c 100644 --- a/sqlparser/lib/src/ast/clauses/limit.dart +++ b/sqlparser/lib/src/ast/clauses/limit.dart @@ -1,6 +1,6 @@ part of '../ast.dart'; -abstract class LimitBase {} +abstract class LimitBase implements AstNode {} class Limit extends AstNode implements LimitBase { Expression count; diff --git a/sqlparser/lib/src/ast/statements/select.dart b/sqlparser/lib/src/ast/statements/select.dart index 634846e2..cf9c9dc4 100644 --- a/sqlparser/lib/src/ast/statements/select.dart +++ b/sqlparser/lib/src/ast/statements/select.dart @@ -10,7 +10,7 @@ class SelectStatement extends Statement with CrudStatement, ResultSet { final List windowDeclarations; final OrderBy orderBy; - final Limit limit; + final LimitBase limit; /// The resolved list of columns returned by this select statements. Not /// available from the parse tree, will be set later by the analyzer. diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index e0ba177f..04ef84f5 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -303,9 +303,11 @@ mixin CrudParser on ParserBase { /// Parses a [Limit] clause, or returns null if there is no limit token after /// the current position. - Limit _limit() { + LimitBase _limit() { if (!_matchOne(TokenType.limit)) return null; + final limitToken = _previous; + // Unintuitive, it's "$amount OFFSET $offset", but "$offset, $amount" // the order changes between the separator tokens. final first = expression(); @@ -313,13 +315,22 @@ mixin CrudParser on ParserBase { if (_matchOne(TokenType.comma)) { final separator = _previous; final count = expression(); - return Limit(count: count, offsetSeparator: separator, offset: first); + return Limit(count: count, offsetSeparator: separator, offset: first) + ..setSpan(limitToken, _previous); } else if (_matchOne(TokenType.offset)) { final separator = _previous; final offset = expression(); - return Limit(count: first, offsetSeparator: separator, offset: offset); + return Limit(count: first, offsetSeparator: separator, offset: offset) + ..setSpan(limitToken, _previous); } else { - return Limit(count: first); + // no offset or comma was parsed (so just LIMIT $expr). In that case, we + // want to provide additional flexibility to the user by interpreting the + // expression as a whole limit clause. + if (first is InlineDartExpression) { + return InlineDartLimit(name: first.name) + ..setSpan(limitToken, _previous); + } + return Limit(count: first)..setSpan(limitToken, _previous); } } diff --git a/sqlparser/lib/src/reader/tokenizer/scanner.dart b/sqlparser/lib/src/reader/tokenizer/scanner.dart index 71218ac2..db5ccc88 100644 --- a/sqlparser/lib/src/reader/tokenizer/scanner.dart +++ b/sqlparser/lib/src/reader/tokenizer/scanner.dart @@ -136,7 +136,7 @@ class Scanner { break; case r'$': final name = _matchColumnName(); - tokens.add(ColonVariableToken(_currentSpan, name)); + tokens.add(DollarSignVariableToken(_currentSpan, name)); break; case ';': _addToken(TokenType.semicolon); diff --git a/sqlparser/test/analysis/variable_indexing_test.dart b/sqlparser/test/analysis/variable_indexing_test.dart index 87e043c8..343d60f4 100644 --- a/sqlparser/test/analysis/variable_indexing_test.dart +++ b/sqlparser/test/analysis/variable_indexing_test.dart @@ -13,7 +13,7 @@ void main() { final select = context.root as SelectStatement; final firstEquals = (select.columns[0] as ExpressionResultColumn).expression as BinaryExpression; - final limit = select.limit; + final limit = select.limit as Limit; expect((firstEquals.left as Variable).resolvedIndex, 1); expect((firstEquals.right as Variable).resolvedIndex, 2); diff --git a/sqlparser/test/parser/inline_dart_test.dart b/sqlparser/test/parser/inline_dart_test.dart new file mode 100644 index 00000000..bd0593c4 --- /dev/null +++ b/sqlparser/test/parser/inline_dart_test.dart @@ -0,0 +1,34 @@ +import 'package:sqlparser/sqlparser.dart'; +import 'package:test/test.dart'; + +import 'utils.dart'; + +void main() { + test('parses limit components', () { + testStatement( + r'SELECT * FROM tbl LIMIT $limit', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + limit: InlineDartLimit(name: 'limit'), + ), + moorMode: true, + ); + }); + + test('parses limit counts as expressions', () { + testStatement( + r'SELECT * FROM tbl LIMIT $amount OFFSET 3', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + limit: Limit( + count: InlineDartExpression(name: 'amount'), + offsetSeparator: token(TokenType.offset), + offset: NumericLiteral(3, token(TokenType.numberLiteral)), + ), + ), + moorMode: true, + ); + }); +} diff --git a/sqlparser/test/parser/utils.dart b/sqlparser/test/parser/utils.dart index b88573d4..440e1fc2 100644 --- a/sqlparser/test/parser/utils.dart +++ b/sqlparser/test/parser/utils.dart @@ -18,8 +18,8 @@ IdentifierToken identifier(String content) { return IdentifierToken(false, fakeSpan(content)); } -void testStatement(String sql, AstNode expected) { - final parsed = SqlEngine().parse(sql).rootNode; +void testStatement(String sql, AstNode expected, {bool moorMode = false}) { + final parsed = SqlEngine(useMoorExtensions: moorMode).parse(sql).rootNode; enforceEqual(parsed, expected); } From 7a8191fe7ed29e50d33884987e6ef6d905989e31 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 09:27:36 +0200 Subject: [PATCH 082/117] Fix moor queries not expanding array vars properly --- moor/example/example.g.dart | 4 ++-- moor/test/data/tables/custom_tables.g.dart | 19 ++++++++++++++++--- moor/test/data/tables/tables.moor | 3 ++- moor/test/data/tables/todos.g.dart | 12 ++++++------ .../lib/src/analyzer/moor/parser.dart | 17 ++++++++++------- .../src/analyzer/sql_queries/sql_parser.dart | 2 +- moor_generator/lib/src/model/sql_query.dart | 2 ++ .../lib/src/writer/queries/query_writer.dart | 18 +++++++++--------- sqlparser/lib/src/ast/ast.dart | 3 ++- sqlparser/lib/src/engine/sql_engine.dart | 10 ++++++++-- sqlparser/lib/src/reader/parser/crud.dart | 5 ++++- 11 files changed, 62 insertions(+), 33 deletions(-) diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index 68eec0dd..b3097fa2 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -848,8 +848,8 @@ abstract class _$Database extends GeneratedDatabase { } Selectable _totalWeightQuery() { - return (operateOn ?? this).customSelectQuery( - ' SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id\n ', + return customSelectQuery( + 'SELECT r.title, SUM(ir.amount) AS total_weight\n FROM recipes r\n INNER JOIN recipe_ingredients ir ON ir.recipe = r.id\n GROUP BY r.id', variables: [], readsFrom: {recipes, ingredientInRecipes}).map(_rowToTotalWeightResult); } diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 83b12f08..462d4c11 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -820,8 +820,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { } Selectable readConfig(String var1) { - return (operateOn ?? this).customSelectQuery( - 'readConfig: SELECT * FROM config WHERE config_key = ?;', + return customSelectQuery('SELECT * FROM config WHERE config_key = ?', variables: [ Variable.withString(var1), ], @@ -830,8 +829,22 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { }).map(_rowToConfigData); } + Selectable readMultiple(List var1) { + var $highestIndex = 1; + final expandedvar1 = $expandVar($highestIndex, var1.length); + $highestIndex += var1.length; + return customSelectQuery( + 'SELECT * FROM config WHERE config_key IN ($expandedvar1)', + variables: [ + for (var $ in var1) Variable.withString($), + ], + readsFrom: { + config + }).map(_rowToConfigData); + } + Future writeConfig(String key, String value) { - return (operateOn ?? this).customInsert( + return customInsert( 'REPLACE INTO config VALUES (:key, :value)', variables: [ Variable.withString(key), diff --git a/moor/test/data/tables/tables.moor b/moor/test/data/tables/tables.moor index 089cdf78..97d18d90 100644 --- a/moor/test/data/tables/tables.moor +++ b/moor/test/data/tables/tables.moor @@ -25,4 +25,5 @@ CREATE TABLE mytable ( sometext TEXT ); -readConfig: SELECT * FROM config WHERE config_key = ?; \ No newline at end of file +readConfig: SELECT * FROM config WHERE config_key = ?; +readMultiple: SELECT * FROM config WHERE config_key IN ?; \ No newline at end of file diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 9d853efc..148602f7 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1313,7 +1313,7 @@ abstract class _$TodoDb extends GeneratedDatabase { } Selectable allTodosWithCategoryQuery() { - return (operateOn ?? this).customSelectQuery( + return customSelectQuery( 'SELECT t.*, c.id as catId, c."desc" as catDesc FROM todos t INNER JOIN categories c ON c.id = t.category', variables: [], readsFrom: { @@ -1331,7 +1331,7 @@ abstract class _$TodoDb extends GeneratedDatabase { } Future deleteTodoById(int var1) { - return (operateOn ?? this).customUpdate( + return customUpdate( 'DELETE FROM todos WHERE id = ?', variables: [ Variable.withInt(var1), @@ -1354,7 +1354,7 @@ abstract class _$TodoDb extends GeneratedDatabase { var $highestIndex = 3; final expandedvar3 = $expandVar($highestIndex, var3.length); $highestIndex += var3.length; - return (operateOn ?? this).customSelectQuery( + return customSelectQuery( 'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1', variables: [ Variable.withString(var1), @@ -1376,7 +1376,7 @@ abstract class _$TodoDb extends GeneratedDatabase { } Selectable searchQuery(int id) { - return (operateOn ?? this).customSelectQuery( + return customSelectQuery( 'SELECT * FROM todos WHERE CASE WHEN -1 = :id THEN 1 ELSE id = :id END', variables: [ Variable.withInt(id), @@ -1402,7 +1402,7 @@ abstract class _$TodoDb extends GeneratedDatabase { } Selectable findCustomQuery() { - return (operateOn ?? this).customSelectQuery( + return customSelectQuery( 'SELECT custom FROM table_without_p_k WHERE some_float < 10', variables: [], readsFrom: {tableWithoutPK}).map(_rowToFindCustomResult); @@ -1472,7 +1472,7 @@ mixin _$SomeDaoMixin on DatabaseAccessor { } Selectable todosForUserQuery(int user) { - return (operateOn ?? this).customSelectQuery( + return customSelectQuery( 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', variables: [ Variable.withInt(user), diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index e4b41563..e7755ecb 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -41,13 +41,16 @@ class MoorParser { final createdTables = createdReaders.map((r) => r.extractTable(step.mapper)).toList(); - return Future.value( - ParsedMoorFile( - result, - declaredTables: createdTables, - queries: queryDeclarations, - imports: importStatements, - ), + final analyzedFile = ParsedMoorFile( + result, + declaredTables: createdTables, + queries: queryDeclarations, + imports: importStatements, ); + for (var decl in queryDeclarations) { + decl.file = analyzedFile; + } + + return Future.value(analyzedFile); } } diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index b2af3755..c05e262d 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -45,7 +45,7 @@ class SqlParser { return; } } else if (query is DeclaredMoorQuery) { - context = _engine.analyzeNode(query.query); + context = _engine.analyzeNode(query.query, query.file.parseResult.sql); declaredInMoor = true; } diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 1d064f2f..32ca979e 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -1,3 +1,4 @@ +import 'package:moor_generator/src/analyzer/runner/results.dart'; import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/used_type_converter.dart'; @@ -33,6 +34,7 @@ class DeclaredDartQuery extends DeclaredQuery { /// available. class DeclaredMoorQuery extends DeclaredQuery { final AstNode query; + ParsedMoorFile file; DeclaredMoorQuery(String name, this.query) : super(name); diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 472db00b..b064157e 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -120,9 +120,7 @@ class QueryWriter { _buffer.write(') {\n'); _writeExpandedDeclarations(); - _buffer - ..write('return (operateOn ?? this).') - ..write('customSelectQuery(${_queryCode()}, '); + _buffer.write('return customSelectQuery(${_queryCode()}, '); _writeVariables(); _buffer.write(', '); _writeReadsFrom(); @@ -172,9 +170,7 @@ class QueryWriter { _buffer.write(') {\n'); _writeExpandedDeclarations(); - _buffer - ..write('return (operateOn ?? this).') - ..write('$implName(${_queryCode()},'); + _buffer.write('return $implName(${_queryCode()},'); _writeVariables(); _buffer.write(','); @@ -285,14 +281,15 @@ class QueryWriter { ..sort((a, b) => a.firstPosition.compareTo(b.firstPosition)); final buffer = StringBuffer("'"); - var lastIndex = 0; + + var lastIndex = query.fromContext.root.firstPosition; for (var sqlVar in vars) { final moorVar = query.variables .singleWhere((f) => f.variable.resolvedIndex == sqlVar.resolvedIndex); if (!moorVar.isArray) continue; - // write everything that comes before this var into the_buffer + // write everything that comes before this var into the buffer final currentIndex = sqlVar.firstPosition; final queryPart = query.sql.substring(lastIndex, currentIndex); buffer.write(escapeForDart(queryPart)); @@ -303,7 +300,10 @@ class QueryWriter { } // write the final part after the last variable, plus the ending ' - buffer..write(escapeForDart(query.sql.substring(lastIndex)))..write("'"); + final lastPosition = query.fromContext.root.lastPosition; + buffer + ..write(escapeForDart(query.sql.substring(lastIndex, lastPosition))) + ..write("'"); return buffer.toString(); } diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index a33dc882..6c7ad204 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -54,7 +54,8 @@ abstract class AstNode { /// nodes. int get firstPosition => first.span.start.offset; - /// The last position that belongs to node, exclusive. Not set for all nodes. + /// The (exclusive) last index of this node in the source. In other words, the + /// first index that is _not_ a part of this node. Not set for all nodes. int get lastPosition => last.span.end.offset; FileSpan get span { diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 0d0af363..1146e819 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -102,8 +102,8 @@ class SqlEngine { /// The analyzer needs to know all the available tables to resolve references /// and result columns, so all known tables should be registered using /// [registerTable] before calling this method. - AnalysisContext analyzeNode(AstNode node) { - final context = AnalysisContext(node, node.span.context); + AnalysisContext analyzeNode(AstNode node, String file) { + final context = AnalysisContext(node, file); _analyzeContext(context); return context; } @@ -185,4 +185,10 @@ class ParseResult { return candidates.toList(); } + + /// Returns the lexeme that created an AST [node] (which should be a child of + /// [rootNode], e.g appear in this result). + String lexemeOfNode(AstNode node) { + return sql.substring(node.firstPosition, node.lastPosition); + } } diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 04ef84f5..75dfd306 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -353,6 +353,8 @@ mixin CrudParser on ParserBase { UpdateStatement _update() { if (!_matchOne(TokenType.update)) return null; + final updateToken = _previous; + FailureMode failureMode; if (_matchOne(TokenType.or)) { failureMode = UpdateStatement.failureModeFromToken(_advance().type); @@ -376,7 +378,8 @@ mixin CrudParser on ParserBase { final where = _where(); return UpdateStatement( - or: failureMode, table: table, set: set, where: where); + or: failureMode, table: table, set: set, where: where) + ..setSpan(updateToken, _previous); } InsertStatement _insertStmt() { From 1d7b4d01fe14b516286c3c66d822eafca53bf9ad Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 10:15:09 +0200 Subject: [PATCH 083/117] Add documentation on Dart expressions --- .../en/docs/Getting started/expressions.md | 78 +++++++++++++++++++ .../docs/Getting started/writing_queries.md | 1 + 2 files changed, 79 insertions(+) create mode 100644 docs/content/en/docs/Getting started/expressions.md diff --git a/docs/content/en/docs/Getting started/expressions.md b/docs/content/en/docs/Getting started/expressions.md new file mode 100644 index 00000000..4bbb4e9d --- /dev/null +++ b/docs/content/en/docs/Getting started/expressions.md @@ -0,0 +1,78 @@ +--- +title: "Expressions" +linkTitle: "Expressions" +description: Deep-dive into what kind of SQL expressions can be written in Dart +weight: 200 +--- + +Expressions are pieces of sql that return a value when the database interprets them. +The Dart API from moor allows you to write most expressions in Dart and then convert +them to sql. Expressions are used in all kinds of situations. For instance, `where` +expects an expression that returns a boolean. + +In most cases, you're writing an expression that combines other expressions. Any +column name is a valid expression, so for most `where` clauses you'll be writing +a expression that wraps a column name in some kind of comparison. + +## Comparisons +Every expression can be compared to a value by using `equals`. If you want to compare +an expression to another expression, you can use `equalsExpr`. For numeric and datetime +expressions, you can also use a variety of methods like `isSmallerThan`, `isSmallerOrEqual` +and so on to compare them: +```dart +// find all animals with less than 5 legs: +(select(animals)..where((a) => a.amountOfLegs.isSmallerThanValue(5))).get(); + +// find all animals who's average livespan is shorter than their amount of legs (poor flies) +(select(animals)..where((a) => a.averageLivespan.isSmallerThan(a.amountOfLegs))); + +Future> findAnimalsByLegs(int legCount) { + return (select(animals)..where((a) => a.legs.equals(legCount))).get(); +} +``` + +## Boolean algebra +You can nest boolean expressions by using the top-level `and`, `or` and `not` functions +exposed by moor: +```dart +// find all animals that aren't mammals and have 4 legs +select(animals)..where((a) => and(not(a.isMammal), a.amountOfLegs.equals(4))) +``` + +## Nullability +To check whether an expression returns null, you can use the top-level `isNull` function, +which takes any expression and returns a boolean expression. The expression returned will +resolve to `true` if the inner expression resolves to null and `false` otherwise. +As you would expect, `isNotNull` works the other way around. + +## Date and Time +For columns and expressions that return a `DateTime`, you can use the top-level +`year`, `month`, `day`, `hour`, `minute` and `second` functions to extract individual +fields from that date: +```dart +select(users)..where((u) => year(u.birthDate).isLessThan(1950)) +``` + +To obtain the current date or the current time as an expression, use the `currentDate` +and `currentDateAndTime` constants provided by moor. + +## `IN` and `NOT IN` +You can check whether an expression is in a list of values by using the `isIn` function: +```dart +select(animals)..where((a) => isIn(a.amountOfLegs, [3, 7, 4, 2])) +``` + +Again, the `isNotIn` function works the other way around. + +## Custom expressions +If you want to inline custom sql into Dart queries, you can use a `CustomExpression` class. +It takes a `sql` parameter that let's you write custom expressions: +```dart +const inactive = CustomExpression("julianday('now') - julianday(last_login) > 60"); +select(users)..where((u) => inactive); +``` + +_Note_: It's easy to write invalid queries by using `CustomExpressions` too much. If you feel like +you need to use them because a feature you use is not available in moor, consider creating an issue +to let us know. If you just prefer sql, you could also take a look at +[compiled sql]({{< ref "../Using SQL/custom_queries.md" >}}) which is typesafe to use. \ No newline at end of file diff --git a/docs/content/en/docs/Getting started/writing_queries.md b/docs/content/en/docs/Getting started/writing_queries.md index 384e75d4..7a19c59f 100644 --- a/docs/content/en/docs/Getting started/writing_queries.md +++ b/docs/content/en/docs/Getting started/writing_queries.md @@ -4,6 +4,7 @@ linkTitle: "Writing queries" description: Learn how to write database queries in pure Dart with moor aliases: - /queries/ +weight: 100 --- {{% pageinfo %}} From f1710987894fcd38428d54bd1b2e34727ecac845 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 11:44:15 +0200 Subject: [PATCH 084/117] Parse Dart placeholders based on their context --- sqlparser/lib/src/ast/ast.dart | 4 +- sqlparser/lib/src/ast/clauses/limit.dart | 3 ++ sqlparser/lib/src/ast/clauses/ordering.dart | 15 +++++-- .../lib/src/ast/expressions/aggregate.dart | 2 +- sqlparser/lib/src/ast/moor/inline_dart.dart | 41 +++++++++++-------- sqlparser/lib/src/ast/statements/select.dart | 2 +- sqlparser/lib/src/reader/parser/crud.dart | 33 +++++++++++---- .../lib/src/reader/parser/expressions.dart | 2 +- .../analysis/reference_resolver_test.dart | 5 ++- sqlparser/test/parser/inline_dart_test.dart | 38 +++++++++++++++-- 10 files changed, 107 insertions(+), 38 deletions(-) diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index 6c7ad204..edeb6953 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -193,7 +193,7 @@ abstract class AstVisitor { T visitMoorFile(MoorFile e); T visitMoorImportStatement(ImportStatement e); T visitMoorDeclaredStatement(DeclaredStatement e); - T visitInlineDartCode(InlineDart e); + T visitDartPlaceholder(DartPlaceholder e); } /// Visitor that walks down the entire tree, visiting all children in order. @@ -316,7 +316,7 @@ class RecursiveVisitor extends AstVisitor { T visitMoorDeclaredStatement(DeclaredStatement e) => visitChildren(e); @override - T visitInlineDartCode(InlineDart e) => visitChildren(e); + T visitDartPlaceholder(DartPlaceholder e) => visitChildren(e); @protected T visitChildren(AstNode e) { diff --git a/sqlparser/lib/src/ast/clauses/limit.dart b/sqlparser/lib/src/ast/clauses/limit.dart index 08e56f4c..b79fa1dd 100644 --- a/sqlparser/lib/src/ast/clauses/limit.dart +++ b/sqlparser/lib/src/ast/clauses/limit.dart @@ -1,5 +1,8 @@ part of '../ast.dart'; +/// Base for limit statements. Without moor extensions, only [Limit] will be +/// parsed. With moor extensions, a [DartLimitPlaceholder] can be emitted as +/// well. abstract class LimitBase implements AstNode {} class Limit extends AstNode implements LimitBase { diff --git a/sqlparser/lib/src/ast/clauses/ordering.dart b/sqlparser/lib/src/ast/clauses/ordering.dart index 860af236..92d4bac1 100644 --- a/sqlparser/lib/src/ast/clauses/ordering.dart +++ b/sqlparser/lib/src/ast/clauses/ordering.dart @@ -1,7 +1,16 @@ part of '../ast.dart'; -class OrderBy extends AstNode { - final List terms; +/// Base for `ORDER BY` clauses. Without moor extensions, ony [OrderBy] will be +/// parsed. Otherwise, [DartOrderByPlaceholder] can be parsed as well. +abstract class OrderByBase extends AstNode {} + +/// Base for a single ordering term that is a part of a [OrderBy]. Without moor +/// extensions, only [OrderingTerm] will be parsed. With moor extensions, a +/// [DartOrderingTermPlaceholder] can be parsed as well. +abstract class OrderingTermBase extends AstNode {} + +class OrderBy extends AstNode implements OrderByBase { + final List terms; OrderBy({this.terms}); @@ -19,7 +28,7 @@ class OrderBy extends AstNode { enum OrderingMode { ascending, descending } -class OrderingTerm extends AstNode { +class OrderingTerm extends AstNode implements OrderingTermBase { final Expression expression; final OrderingMode orderingMode; diff --git a/sqlparser/lib/src/ast/expressions/aggregate.dart b/sqlparser/lib/src/ast/expressions/aggregate.dart index cb6feee6..eadefde8 100644 --- a/sqlparser/lib/src/ast/expressions/aggregate.dart +++ b/sqlparser/lib/src/ast/expressions/aggregate.dart @@ -74,7 +74,7 @@ class NamedWindowDeclaration with Referencable { class WindowDefinition extends AstNode { final String baseWindowName; final List partitionBy; - final OrderBy orderBy; + final OrderByBase orderBy; final FrameSpec frameSpec; WindowDefinition( diff --git a/sqlparser/lib/src/ast/moor/inline_dart.dart b/sqlparser/lib/src/ast/moor/inline_dart.dart index 9efc6b30..d6f738ce 100644 --- a/sqlparser/lib/src/ast/moor/inline_dart.dart +++ b/sqlparser/lib/src/ast/moor/inline_dart.dart @@ -8,40 +8,45 @@ part of '../ast.dart'; /// 1. expressions: Any expression can be used for moor: `SELECT * FROM table /// = $expr`. Generated code will write this as an `Expression` class from /// moor. -/// 2. limits -/// 3. A single order-by clause -/// 4. A list of order-by clauses -abstract class InlineDart extends AstNode { +/// 2. limits, which will be exposed as a `Limit` component from moor +/// 3. A single order-by clause, which will be exposed as a `OrderingTerm` from +/// moor. +/// 4. A list of order-by clauses, which will be exposed as a `OrderBy` from +/// moor. +abstract class DartPlaceholder extends AstNode { final String name; DollarSignVariableToken token; - InlineDart._(this.name); + DartPlaceholder._(this.name); @override final Iterable childNodes = const Iterable.empty(); @override - T accept(AstVisitor visitor) => visitor.visitInlineDartCode(this); + T accept(AstVisitor visitor) => visitor.visitDartPlaceholder(this); - bool _dartEquals(covariant InlineDart other); + bool _dartEquals(covariant DartPlaceholder other) => true; @override - bool contentEquals(InlineDart other) { + bool contentEquals(DartPlaceholder other) { return other.name == name && other._dartEquals(other); } } -class InlineDartExpression extends InlineDart implements Expression { - InlineDartExpression({@required String name}) : super._(name); - - @override - bool _dartEquals(InlineDartExpression other) => true; +class DartExpressionPlaceholder extends DartPlaceholder implements Expression { + DartExpressionPlaceholder({@required String name}) : super._(name); } -class InlineDartLimit extends InlineDart implements LimitBase { - InlineDartLimit({@required String name}) : super._(name); - - @override - bool _dartEquals(InlineDartLimit other) => true; +class DartLimitPlaceholder extends DartPlaceholder implements LimitBase { + DartLimitPlaceholder({@required String name}) : super._(name); +} + +class DartOrderingTermPlaceholder extends DartPlaceholder + implements OrderingTermBase { + DartOrderingTermPlaceholder({@required String name}) : super._(name); +} + +class DartOrderByPlaceholder extends DartPlaceholder implements OrderByBase { + DartOrderByPlaceholder({@required String name}) : super._(name); } diff --git a/sqlparser/lib/src/ast/statements/select.dart b/sqlparser/lib/src/ast/statements/select.dart index cf9c9dc4..a3ed4d7d 100644 --- a/sqlparser/lib/src/ast/statements/select.dart +++ b/sqlparser/lib/src/ast/statements/select.dart @@ -9,7 +9,7 @@ class SelectStatement extends Statement with CrudStatement, ResultSet { final GroupBy groupBy; final List windowDeclarations; - final OrderBy orderBy; + final OrderByBase orderBy; final LimitBase limit; /// The resolved list of columns returned by this select statements. Not diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 75dfd306..5fbcacd7 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -272,22 +272,41 @@ mixin CrudParser on ParserBase { return declarations; } - OrderBy _orderBy() { + OrderByBase _orderBy() { if (_matchOne(TokenType.order)) { _consume(TokenType.by, 'Expected "BY" after "ORDER" token'); - final terms = []; + final terms = []; do { terms.add(_orderingTerm()); } while (_matchOne(TokenType.comma)); + + // If we only hit a single ordering term and that term is a Dart + // placeholder, we can upgrade that term to a full order by placeholder. + // This gives users more control at runtime (they can specify multiple + // terms). + if (terms.length == 1 && terms.single is DartOrderingTermPlaceholder) { + final termPlaceholder = terms.single as DartOrderingTermPlaceholder; + return DartOrderByPlaceholder(name: termPlaceholder.name); + } + return OrderBy(terms: terms); } return null; } - OrderingTerm _orderingTerm() { + OrderingTermBase _orderingTerm() { final expr = expression(); + final mode = _orderingModeOrNull(); - return OrderingTerm(expression: expr, orderingMode: _orderingModeOrNull()); + // if there is no ASC or DESC after a Dart placeholder, we can upgrade the + // expression to an ordering term placeholder and let users define the mode + // at runtime. + if (mode == null && expr is DartExpressionPlaceholder) { + return DartOrderingTermPlaceholder(name: expr.name) + ..setSpan(expr.first, expr.last); + } + + return OrderingTerm(expression: expr, orderingMode: mode); } @override @@ -326,8 +345,8 @@ mixin CrudParser on ParserBase { // no offset or comma was parsed (so just LIMIT $expr). In that case, we // want to provide additional flexibility to the user by interpreting the // expression as a whole limit clause. - if (first is InlineDartExpression) { - return InlineDartLimit(name: first.name) + if (first is DartExpressionPlaceholder) { + return DartLimitPlaceholder(name: first.name) ..setSpan(limitToken, _previous); } return Limit(count: first)..setSpan(limitToken, _previous); @@ -457,7 +476,7 @@ mixin CrudParser on ParserBase { final leftParen = _previous; String baseWindowName; - OrderBy orderBy; + OrderByBase orderBy; final partitionBy = []; if (_matchOne(TokenType.identifier)) { diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index 48d0a624..e256ad7f 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -322,7 +322,7 @@ mixin ExpressionParser on ParserBase { case TokenType.dollarSignVariable: if (enableMoorExtensions) { final typedToken = token as DollarSignVariableToken; - return InlineDartExpression(name: typedToken.name) + return DartExpressionPlaceholder(name: typedToken.name) ..token = typedToken ..setSpan(token, token); } diff --git a/sqlparser/test/analysis/reference_resolver_test.dart b/sqlparser/test/analysis/reference_resolver_test.dart index 6e0009ca..2e49eac1 100644 --- a/sqlparser/test/analysis/reference_resolver_test.dart +++ b/sqlparser/test/analysis/reference_resolver_test.dart @@ -46,8 +46,9 @@ void main() { expect(context.errors, isEmpty); final select = context.root as SelectStatement; - final orderingTerm = select.orderBy.terms.single.expression as Reference; - final resolved = orderingTerm.resolved as ExpressionColumn; + final term = (select.orderBy as OrderBy).terms.single as OrderingTerm; + final expression = term.expression as Reference; + final resolved = expression.resolved as ExpressionColumn; enforceEqual( resolved.expression, diff --git a/sqlparser/test/parser/inline_dart_test.dart b/sqlparser/test/parser/inline_dart_test.dart index bd0593c4..16cb9b1f 100644 --- a/sqlparser/test/parser/inline_dart_test.dart +++ b/sqlparser/test/parser/inline_dart_test.dart @@ -10,20 +10,20 @@ void main() { SelectStatement( columns: [StarResultColumn(null)], from: [TableReference('tbl', null)], - limit: InlineDartLimit(name: 'limit'), + limit: DartLimitPlaceholder(name: 'limit'), ), moorMode: true, ); }); - test('parses limit counts as expressions', () { + test('parses limit count as expressions', () { testStatement( r'SELECT * FROM tbl LIMIT $amount OFFSET 3', SelectStatement( columns: [StarResultColumn(null)], from: [TableReference('tbl', null)], limit: Limit( - count: InlineDartExpression(name: 'amount'), + count: DartExpressionPlaceholder(name: 'amount'), offsetSeparator: token(TokenType.offset), offset: NumericLiteral(3, token(TokenType.numberLiteral)), ), @@ -31,4 +31,36 @@ void main() { moorMode: true, ); }); + + test('parses ordering terms and ordering expressions', () { + testStatement( + r'SELECT * FROM tbl ORDER BY $term, $expr DESC', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + orderBy: OrderBy( + terms: [ + DartOrderingTermPlaceholder(name: 'term'), + OrderingTerm( + expression: DartExpressionPlaceholder(name: 'expr'), + orderingMode: OrderingMode.descending, + ), + ], + ), + ), + moorMode: true, + ); + }); + + test('parses full order by placeholders', () { + testStatement( + r'SELECT * FROM tbl ORDER BY $order', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + orderBy: DartOrderByPlaceholder(name: 'order'), + ), + moorMode: true, + ); + }); } From bf7c9feddfb30539ab597702ee7378c306274807 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 12:26:58 +0200 Subject: [PATCH 085/117] Write parameter declarations for Dart placeholders --- moor/test/data/tables/custom_tables.g.dart | 4 +- moor/test/data/tables/tables.moor | 2 +- .../analyzer/sql_queries/query_handler.dart | 10 +-- .../analyzer/sql_queries/type_mapping.dart | 27 +++++++ moor_generator/lib/src/model/sql_query.dart | 71 +++++++++++++++++-- .../lib/src/writer/queries/query_writer.dart | 15 +++- sqlparser/lib/src/ast/moor/inline_dart.dart | 18 +++++ 7 files changed, 130 insertions(+), 17 deletions(-) diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 462d4c11..2d2f70f3 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -829,12 +829,12 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { }).map(_rowToConfigData); } - Selectable readMultiple(List var1) { + Selectable readMultiple(List var1, OrderBy clause) { var $highestIndex = 1; final expandedvar1 = $expandVar($highestIndex, var1.length); $highestIndex += var1.length; return customSelectQuery( - 'SELECT * FROM config WHERE config_key IN ($expandedvar1)', + 'SELECT * FROM config WHERE config_key IN ($expandedvar1) ORDER BY \$clause', variables: [ for (var $ in var1) Variable.withString($), ], diff --git a/moor/test/data/tables/tables.moor b/moor/test/data/tables/tables.moor index 97d18d90..df9164be 100644 --- a/moor/test/data/tables/tables.moor +++ b/moor/test/data/tables/tables.moor @@ -26,4 +26,4 @@ CREATE TABLE mytable ( ); readConfig: SELECT * FROM config WHERE config_key = ?; -readMultiple: SELECT * FROM config WHERE config_key IN ?; \ No newline at end of file +readMultiple: SELECT * FROM config WHERE config_key IN ? ORDER BY $clause; \ No newline at end of file diff --git a/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart index 0e5e9454..0b0b23a8 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart @@ -17,6 +17,7 @@ class QueryHandler { Set
_foundTables; List _foundVariables; + List _foundPlaceholders; SelectStatement get _select => context.root as SelectStatement; @@ -24,6 +25,7 @@ class QueryHandler { SqlQuery handle() { _foundVariables = mapper.extractVariables(context); + _foundPlaceholders = mapper.extractPlaceholders(context); _verifyNoSkippedIndexes(); final query = _mapToMoor(); @@ -45,7 +47,7 @@ class QueryHandler { return _handleUpdate(); } else { throw StateError( - 'Unexpected sql: Got $root, expected a select or update statement'); + 'Unexpected sql: Got $root, expected insert, select, update or delete'); } } @@ -56,7 +58,7 @@ class QueryHandler { final isInsert = context.root is InsertStatement; - return UpdatingQuery(name, context, _foundVariables, + return UpdatingQuery(name, context, _foundVariables, _foundPlaceholders, _foundTables.map(mapper.tableToMoor).toList(), isInsert: isInsert); } @@ -67,8 +69,8 @@ class QueryHandler { _foundTables = tableFinder.foundTables; final moorTables = _foundTables.map(mapper.tableToMoor).toList(); - return SqlSelectQuery( - name, context, _foundVariables, moorTables, _inferResultSet()); + return SqlSelectQuery(name, context, _foundVariables, _foundPlaceholders, + moorTables, _inferResultSet()); } InferredResultSet _inferResultSet() { diff --git a/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart b/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart index 48f1d31a..31ec1cc5 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart @@ -123,6 +123,33 @@ class TypeMapper { return foundVariables; } + List extractPlaceholders(AnalysisContext context) { + final placeholders = + context.root.allDescendants.whereType().toList(); + final found = []; + + for (var placeholder in placeholders) { + ColumnType columnType; + final name = placeholder.name; + + final type = placeholder.when( + isExpression: (e) { + final foundType = context.typeOf(e); + if (foundType.type != null) { + columnType = resolvedToMoor(foundType.type); + } + return DartPlaceholderType.expression; + }, + isLimit: (_) => DartPlaceholderType.limit, + isOrderBy: (_) => DartPlaceholderType.orderBy, + isOrderingTerm: (_) => DartPlaceholderType.orderByTerm, + ); + + found.add(FoundDartPlaceholder(type, columnType, name)); + } + return found; + } + SpecifiedTable tableToMoor(Table table) { return _engineTablesToSpecified[table]; } diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 32ca979e..0223268a 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -76,7 +76,14 @@ abstract class SqlQuery { /// about the variables that appear after that array. final List variables; - SqlQuery(this.name, this.fromContext, this.variables); + /// The placeholders in this query which are bound and converted to sql at + /// runtime. For instance, in `SELECT * FROM tbl WHERE $expr`, the `expr` is + /// going to be a [FoundDartPlaceholder] with the type + /// [DartPlaceholderType.expression] and [ColumnType.boolean]. We will + /// generate a method which has a `Expression expr` parameter. + final List placeholders; + + SqlQuery(this.name, this.fromContext, this.variables, this.placeholders); } class SqlSelectQuery extends SqlQuery { @@ -90,19 +97,28 @@ class SqlSelectQuery extends SqlQuery { return '${ReCase(name).pascalCase}Result'; } - SqlSelectQuery(String name, AnalysisContext fromContext, - List variables, this.readsFrom, this.resultSet) - : super(name, fromContext, variables); + SqlSelectQuery( + String name, + AnalysisContext fromContext, + List variables, + List placeholders, + this.readsFrom, + this.resultSet) + : super(name, fromContext, variables, placeholders); } class UpdatingQuery extends SqlQuery { final List updates; final bool isInsert; - UpdatingQuery(String name, AnalysisContext fromContext, - List variables, this.updates, + UpdatingQuery( + String name, + AnalysisContext fromContext, + List variables, + List placeholders, + this.updates, {this.isInsert = false}) - : super(name, fromContext, variables); + : super(name, fromContext, variables, placeholders); } class InferredResultSet { @@ -198,3 +214,44 @@ class FoundVariable { } } } + +enum DartPlaceholderType { + expression, + limit, + orderByTerm, + orderBy, +} + +/// A Dart placeholder that will be bound at runtime. +class FoundDartPlaceholder { + final DartPlaceholderType type; + + /// If [type] is [DartPlaceholderType.expression] and the expression could be + /// resolved, this is the type of that expression. + final ColumnType columnType; + + final String name; + + FoundDartPlaceholder(this.type, this.columnType, this.name); + + /// The type of this parameter on a generated method. + String get parameterType { + switch (type) { + case DartPlaceholderType.expression: + if (columnType == null) return 'Expression'; + + final dartType = dartTypeNames[columnType]; + final sqlImplType = sqlTypes[columnType]; + return 'Expression<$dartType, $sqlImplType>'; + break; + case DartPlaceholderType.limit: + return 'Limit'; + case DartPlaceholderType.orderByTerm: + return 'OrderingTerm'; + case DartPlaceholderType.orderBy: + return 'OrderBy'; + } + + throw AssertionError('cant happen, all branches covered'); + } +} diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index b064157e..9b4e426c 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -180,14 +180,19 @@ class QueryWriter { } void _writeParameters() { - final paramList = query.variables.map((v) { + final variableParams = query.variables.map((v) { var dartType = dartTypeNames[v.type]; if (v.isArray) { dartType = 'List<$dartType>'; } return '$dartType ${v.dartParameterName}'; - }).join(', '); + }); + final placeholderParams = query.placeholders.map((p) { + return '${p.parameterType} ${p.name}'; + }); + + final paramList = variableParams.followedBy(placeholderParams).join(', '); _buffer.write(paramList); } @@ -195,7 +200,11 @@ class QueryWriter { /// assuming that for each parameter, a variable with the same name exists /// in the current scope. void _writeUseParameters() { - _buffer.write(query.variables.map((v) => v.dartParameterName).join(', ')); + final parameters = query.variables + .map((v) => v.dartParameterName) + .followedBy(query.placeholders.map((p) => p.name)); + + _buffer.write(parameters.join(', ')); } // Some notes on parameters and generating query code: diff --git a/sqlparser/lib/src/ast/moor/inline_dart.dart b/sqlparser/lib/src/ast/moor/inline_dart.dart index d6f738ce..031249ef 100644 --- a/sqlparser/lib/src/ast/moor/inline_dart.dart +++ b/sqlparser/lib/src/ast/moor/inline_dart.dart @@ -32,6 +32,24 @@ abstract class DartPlaceholder extends AstNode { bool contentEquals(DartPlaceholder other) { return other.name == name && other._dartEquals(other); } + + T when( + {T Function(DartExpressionPlaceholder) isExpression, + T Function(DartLimitPlaceholder) isLimit, + T Function(DartOrderingTermPlaceholder) isOrderingTerm, + T Function(DartOrderByPlaceholder) isOrderBy}) { + if (this is DartExpressionPlaceholder) { + return isExpression?.call(this as DartExpressionPlaceholder); + } else if (this is DartLimitPlaceholder) { + return isLimit?.call(this as DartLimitPlaceholder); + } else if (this is DartOrderingTermPlaceholder) { + return isOrderingTerm?.call(this as DartOrderingTermPlaceholder); + } else if (this is DartOrderByPlaceholder) { + return isOrderBy?.call(this as DartOrderByPlaceholder); + } + + throw AssertionError('Invalid placeholder: $runtimeType'); + } } class DartExpressionPlaceholder extends DartPlaceholder implements Expression { From 349b245089e9e510ea9fc5e4b4ede037c0fb8ebd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 16:42:24 +0200 Subject: [PATCH 086/117] Write implementation for Dart placeholders --- .../lib/src/runtime/components/component.dart | 11 +- moor/lib/src/runtime/components/order_by.dart | 6 +- moor/lib/src/runtime/database.dart | 16 ++ .../src/runtime/expressions/variables.dart | 24 +-- moor/test/data/tables/custom_tables.g.dart | 10 +- moor/test/data/tables/todos.g.dart | 6 +- .../analyzer/sql_queries/query_handler.dart | 14 +- .../analyzer/sql_queries/type_mapping.dart | 173 ++++++++++++------ moor_generator/lib/src/model/sql_query.dart | 52 +++--- .../lib/src/writer/queries/query_writer.dart | 155 ++++++++++------ .../sql_queries/type_mapping_test.dart | 37 ++++ sqlparser/lib/src/analysis/context.dart | 8 + sqlparser/lib/src/reader/parser/crud.dart | 3 +- 13 files changed, 349 insertions(+), 166 deletions(-) create mode 100644 moor_generator/test/analyzer/sql_queries/type_mapping_test.dart diff --git a/moor/lib/src/runtime/components/component.dart b/moor/lib/src/runtime/components/component.dart index e98ac535..41c655e8 100644 --- a/moor/lib/src/runtime/components/component.dart +++ b/moor/lib/src/runtime/components/component.dart @@ -28,8 +28,16 @@ class GenerationContext { final QueryExecutor executor; final List _boundVariables = []; + + /// The values of [introducedVariables] that will be sent to the underlying + /// engine. List get boundVariables => _boundVariables; + /// All variables ("?" in sql) that were added to this context. + final List introducedVariables = []; + + int get amountOfVariables => boundVariables.length; + /// The string buffer contains the sql query as it's being constructed. final StringBuffer buffer = StringBuffer(); @@ -49,7 +57,8 @@ class GenerationContext { /// that the prepared statement can be executed with the variable. The value /// must be a type that is supported by the sqflite library. A list of /// supported types can be found [here](https://github.com/tekartik/sqflite#supported-sqlite-types). - void introduceVariable(dynamic value) { + void introduceVariable(Variable v, dynamic value) { + introducedVariables.add(v); _boundVariables.add(value); } diff --git a/moor/lib/src/runtime/components/order_by.dart b/moor/lib/src/runtime/components/order_by.dart index eca74fc8..49cb4593 100644 --- a/moor/lib/src/runtime/components/order_by.dart +++ b/moor/lib/src/runtime/components/order_by.dart @@ -44,10 +44,12 @@ class OrderBy extends Component { OrderBy(this.terms); @override - void writeInto(GenerationContext context) { + void writeInto(GenerationContext context, {bool writeOrderBy = true}) { var first = true; - context.buffer.write('ORDER BY '); + if (writeOrderBy) { + context.buffer.write('ORDER BY '); + } for (var term in terms) { if (first) { diff --git a/moor/lib/src/runtime/database.dart b/moor/lib/src/runtime/database.dart index 3c246668..cb307d7f 100644 --- a/moor/lib/src/runtime/database.dart +++ b/moor/lib/src/runtime/database.dart @@ -320,6 +320,22 @@ mixin QueryEngine on DatabaseConnectionUser { QueryEngine engine, Future Function() calculation) { return runZoned(calculation, zoneValues: {_zoneRootUserKey: engine}); } + + /// Will be used by generated code to resolve inline Dart expressions in sql. + @protected + GenerationContext $write(Component component) { + final context = GenerationContext.fromDb(this); + + // we don't want ORDER BY clauses to write the ORDER BY tokens because those + // are already declared in sql + if (component is OrderBy) { + component.writeInto(context, writeOrderBy: false); + } else { + component.writeInto(context); + } + + return context; + } } /// A base class for all generated databases. diff --git a/moor/lib/src/runtime/expressions/variables.dart b/moor/lib/src/runtime/expressions/variables.dart index e91061c1..f4dd2373 100644 --- a/moor/lib/src/runtime/expressions/variables.dart +++ b/moor/lib/src/runtime/expressions/variables.dart @@ -45,12 +45,18 @@ class Variable> extends Expression { /// database engine. For instance, a [DateTime] will me mapped to its unix /// timestamp. dynamic mapToSimpleValue(GenerationContext context) { - return _mapToSimpleValue(context, value); + final type = context.typeSystem.forDartType(); + return type.mapToSqlVariable(value); } @override void writeInto(GenerationContext context) { - _writeVariableIntoContext(context, value); + if (value != null) { + context.buffer.write('?'); + context.introduceVariable(this, mapToSimpleValue(context)); + } else { + context.buffer.write('NULL'); + } } } @@ -71,17 +77,3 @@ class Constant> extends Expression { context.buffer.write(type.mapToSqlConstant(value)); } } - -void _writeVariableIntoContext(GenerationContext context, T value) { - if (value != null) { - context.buffer.write('?'); - context.introduceVariable(_mapToSimpleValue(context, value)); - } else { - context.buffer.write('NULL'); - } -} - -dynamic _mapToSimpleValue(GenerationContext context, T value) { - final type = context.typeSystem.forDartType(); - return type.mapToSqlVariable(value); -} diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 2d2f70f3..137625e1 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -830,11 +830,13 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { } Selectable readMultiple(List var1, OrderBy clause) { - var $highestIndex = 1; - final expandedvar1 = $expandVar($highestIndex, var1.length); - $highestIndex += var1.length; + var $arrayStartIndex = 1; + final expandedvar1 = $expandVar($arrayStartIndex, var1.length); + $arrayStartIndex += var1.length; + final generatedclause = $write(clause); + $arrayStartIndex += generatedclause.amountOfVariables; return customSelectQuery( - 'SELECT * FROM config WHERE config_key IN ($expandedvar1) ORDER BY \$clause', + 'SELECT * FROM config WHERE config_key IN ($expandedvar1) ORDER BY ${generatedclause.sql}', variables: [ for (var $ in var1) Variable.withString($), ], diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 148602f7..da9033ff 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1351,9 +1351,9 @@ abstract class _$TodoDb extends GeneratedDatabase { } Selectable withInQuery(String var1, String var2, List var3) { - var $highestIndex = 3; - final expandedvar3 = $expandVar($highestIndex, var3.length); - $highestIndex += var3.length; + var $arrayStartIndex = 3; + final expandedvar3 = $expandVar($arrayStartIndex, var3.length); + $arrayStartIndex += var3.length; return customSelectQuery( 'SELECT * FROM todos WHERE title = ?2 OR id IN ($expandedvar3) OR title = ?1', variables: [ diff --git a/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart index 0b0b23a8..7ac3253a 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart @@ -16,16 +16,16 @@ class QueryHandler { final TypeMapper mapper; Set
_foundTables; - List _foundVariables; - List _foundPlaceholders; + List _foundElements; + Iterable get _foundVariables => + _foundElements.whereType(); SelectStatement get _select => context.root as SelectStatement; QueryHandler(this.name, this.context, this.mapper); SqlQuery handle() { - _foundVariables = mapper.extractVariables(context); - _foundPlaceholders = mapper.extractPlaceholders(context); + _foundElements = mapper.extractElements(context); _verifyNoSkippedIndexes(); final query = _mapToMoor(); @@ -58,7 +58,7 @@ class QueryHandler { final isInsert = context.root is InsertStatement; - return UpdatingQuery(name, context, _foundVariables, _foundPlaceholders, + return UpdatingQuery(name, context, _foundElements, _foundTables.map(mapper.tableToMoor).toList(), isInsert: isInsert); } @@ -69,8 +69,8 @@ class QueryHandler { _foundTables = tableFinder.foundTables; final moorTables = _foundTables.map(mapper.tableToMoor).toList(); - return SqlSelectQuery(name, context, _foundVariables, _foundPlaceholders, - moorTables, _inferResultSet()); + return SqlSelectQuery( + name, context, _foundElements, moorTables, _inferResultSet()); } InferredResultSet _inferResultSet() { diff --git a/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart b/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart index 31ec1cc5..4a655c32 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/type_mapping.dart @@ -72,82 +72,141 @@ class TypeMapper { throw StateError('Unexpected type: $type'); } - List extractVariables(AnalysisContext ctx) { + /// Extracts variables and Dart templates from the [ctx]. Variables are + /// sorted by their ascending index. Placeholders are sorted by the position + /// they have in the query. When comparing variables and placeholders, the + /// variable comes first if the first variable with the same index appears + /// before the placeholder. + /// + /// Additionally, the following assumptions can be made if this method returns + /// without throwing: + /// - array variables don't have an explicit index + /// - if an explicitly indexed variable appears AFTER an array variable or + /// a Dart placeholder, its indexed is LOWER than that element. This means + /// that elements can be expanded into multiple variables without breaking + /// variables that appear after them. + List extractElements(AnalysisContext ctx) { // this contains variable references. For instance, SELECT :a = :a would // contain two entries, both referring to the same variable. To do that, // we use the fact that each variable has a unique index. - final usedVars = ctx.root.allDescendants.whereType().toList() - ..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex)); + final variables = ctx.root.allDescendants.whereType().toList(); + final placeholders = + ctx.root.allDescendants.whereType().toList(); - final foundVariables = []; + final merged = _mergeVarsAndPlaceholders(variables, placeholders); + + final foundElements = []; // we don't allow variables with an explicit index after an array. For // instance: SELECT * FROM t WHERE id IN ? OR id = ?2. The reason this is // not allowed is that we expand the first arg into multiple vars at runtime - // which would break the index. + // which would break the index. The initial high values can be arbitrary. + // We've chosen 999 because most sqlite binaries don't allow more variables. var maxIndex = 999; var currentIndex = 0; - for (var used in usedVars) { - if (used.resolvedIndex == currentIndex) { - continue; // already handled - } + for (var used in merged) { + if (used is Variable) { + if (used.resolvedIndex == currentIndex) { + continue; // already handled, we only report a single variable / index + } - currentIndex = used.resolvedIndex; - final name = (used is ColonNamedVariable) ? used.name : null; - final explicitIndex = - (used is NumberedVariable) ? used.explicitIndex : null; - final internalType = ctx.typeOf(used); - final type = resolvedToMoor(internalType.type); - final isArray = internalType.type?.isArray ?? false; + currentIndex = used.resolvedIndex; + final name = (used is ColonNamedVariable) ? used.name : null; + final explicitIndex = + (used is NumberedVariable) ? used.explicitIndex : null; + final internalType = ctx.typeOf(used); + final type = resolvedToMoor(internalType.type); + final isArray = internalType.type?.isArray ?? false; - if (explicitIndex != null && currentIndex >= maxIndex) { - throw ArgumentError( - 'Cannot have a variable with an index lower than that of an array ' - 'appearing after an array!'); - } + if (explicitIndex != null && currentIndex >= maxIndex) { + throw ArgumentError( + 'Cannot have a variable with an index lower than that of an array ' + 'appearing after an array!'); + } - foundVariables - .add(FoundVariable(currentIndex, name, type, used, isArray)); + foundElements + .add(FoundVariable(currentIndex, name, type, used, isArray)); - // arrays cannot be indexed explicitly because they're expanded into - // multiple variables when executed - if (isArray && explicitIndex != null) { - throw ArgumentError( - 'Cannot use an array variable with an explicit index'); - } - if (isArray) { - maxIndex = used.resolvedIndex; + // arrays cannot be indexed explicitly because they're expanded into + // multiple variables when executed + if (isArray && explicitIndex != null) { + throw ArgumentError( + 'Cannot use an array variable with an explicit index'); + } + if (isArray) { + maxIndex = used.resolvedIndex; + } + } else if (used is DartPlaceholder) { + // we don't what index this placeholder has, so we can't allow _any_ + // explicitly indexed variables coming after this + maxIndex = 0; + foundElements.add(_extractPlaceholder(ctx, used)); } } - - return foundVariables; + return foundElements; } - List extractPlaceholders(AnalysisContext context) { - final placeholders = - context.root.allDescendants.whereType().toList(); - final found = []; - - for (var placeholder in placeholders) { - ColumnType columnType; - final name = placeholder.name; - - final type = placeholder.when( - isExpression: (e) { - final foundType = context.typeOf(e); - if (foundType.type != null) { - columnType = resolvedToMoor(foundType.type); - } - return DartPlaceholderType.expression; - }, - isLimit: (_) => DartPlaceholderType.limit, - isOrderBy: (_) => DartPlaceholderType.orderBy, - isOrderingTerm: (_) => DartPlaceholderType.orderByTerm, - ); - - found.add(FoundDartPlaceholder(type, columnType, name)); + /// Merges [vars] and [placeholders] into a list that satisfies the order + /// described in [extractElements]. + List _mergeVarsAndPlaceholders( + List vars, List placeholders) { + final groupVarsByIndex = >{}; + for (var variable in vars) { + groupVarsByIndex + .putIfAbsent(variable.resolvedIndex, () => []) + .add(variable); } - return found; + // sort each group by index + for (var group in groupVarsByIndex.values) { + group..sort((a, b) => a.resolvedIndex.compareTo(b.resolvedIndex)); + } + + int Function(dynamic, dynamic) comparer; + comparer = (dynamic a, dynamic b) { + if (a is Variable && b is Variable) { + // variables are sorted by their index + return a.resolvedIndex.compareTo(b.resolvedIndex); + } else if (a is DartPlaceholder && b is DartPlaceholder) { + // placeholders by their position + return AnalysisContext.compareNodesByOrder(a, b); + } else { + // ok, one of them is a variable, the other one is a placeholder. Let's + // assume a is the variable. If not, we just switch results. + if (a is Variable) { + final placeholderB = b as DartPlaceholder; + final firstWithSameIndex = groupVarsByIndex[a.resolvedIndex].first; + + return firstWithSameIndex.firstPosition + .compareTo(placeholderB.firstPosition); + } else { + return -comparer(b, a); + } + } + }; + + final list = vars.cast().followedBy(placeholders).toList(); + return list..sort(comparer); + } + + FoundDartPlaceholder _extractPlaceholder( + AnalysisContext context, DartPlaceholder placeholder) { + ColumnType columnType; + final name = placeholder.name; + + final type = placeholder.when( + isExpression: (e) { + final foundType = context.typeOf(e); + if (foundType.type != null) { + columnType = resolvedToMoor(foundType.type); + } + return DartPlaceholderType.expression; + }, + isLimit: (_) => DartPlaceholderType.limit, + isOrderBy: (_) => DartPlaceholderType.orderBy, + isOrderingTerm: (_) => DartPlaceholderType.orderByTerm, + ); + + return FoundDartPlaceholder(type, columnType, name)..astNode = placeholder; } SpecifiedTable tableToMoor(Table table) { diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index 0223268a..e68317bc 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -74,16 +74,23 @@ abstract class SqlQuery { /// if their index is lower than that of the array (e.g `a = ?2 AND b IN ? /// AND c IN ?1`. In other words, we can expand an array without worrying /// about the variables that appear after that array. - final List variables; + List variables; /// The placeholders in this query which are bound and converted to sql at /// runtime. For instance, in `SELECT * FROM tbl WHERE $expr`, the `expr` is /// going to be a [FoundDartPlaceholder] with the type /// [DartPlaceholderType.expression] and [ColumnType.boolean]. We will /// generate a method which has a `Expression expr` parameter. - final List placeholders; + List placeholders; - SqlQuery(this.name, this.fromContext, this.variables, this.placeholders); + /// Union of [variables] and [elements], but in the order in which they + /// appear inside the query. + final List elements; + + SqlQuery(this.name, this.fromContext, this.elements) { + variables = elements.whereType().toList(); + placeholders = elements.whereType().toList(); + } } class SqlSelectQuery extends SqlQuery { @@ -97,28 +104,19 @@ class SqlSelectQuery extends SqlQuery { return '${ReCase(name).pascalCase}Result'; } - SqlSelectQuery( - String name, - AnalysisContext fromContext, - List variables, - List placeholders, - this.readsFrom, - this.resultSet) - : super(name, fromContext, variables, placeholders); + SqlSelectQuery(String name, AnalysisContext fromContext, + List elements, this.readsFrom, this.resultSet) + : super(name, fromContext, elements); } class UpdatingQuery extends SqlQuery { final List updates; final bool isInsert; - UpdatingQuery( - String name, - AnalysisContext fromContext, - List variables, - List placeholders, - this.updates, + UpdatingQuery(String name, AnalysisContext fromContext, + List elements, this.updates, {this.isInsert = false}) - : super(name, fromContext, variables, placeholders); + : super(name, fromContext, elements); } class InferredResultSet { @@ -177,12 +175,19 @@ class ResultColumn { ResultColumn(this.name, this.type, this.nullable, {this.converter}); } +/// Something in the query that needs special attention when generating code, +/// such as variables or Dart placeholders. +abstract class FoundElement { + String get dartParameterName; +} + /// A semantic interpretation of a [Variable] in a sql statement. -class FoundVariable { +class FoundVariable extends FoundElement { /// The (unique) index of this variable in the sql query. For instance, the /// query `SELECT * FROM tbl WHERE a = ? AND b = :xyz OR c = :xyz` contains /// three [Variable]s in its AST, but only two [FoundVariable]s, where the - /// `?` will have index 1 and (both) `:xyz` variables will have index 2. + /// `?` will have index 1 and (both) `:xyz` variables will have index 2. We + /// only report one [FoundVariable] per index. int index; /// The name of this variable, or null if it's not a named variable. @@ -206,6 +211,7 @@ class FoundVariable { assert(variable.resolvedIndex == index); } + @override String get dartParameterName { if (name != null) { return name.replaceAll(_illegalChars, ''); @@ -223,7 +229,7 @@ enum DartPlaceholderType { } /// A Dart placeholder that will be bound at runtime. -class FoundDartPlaceholder { +class FoundDartPlaceholder extends FoundElement { final DartPlaceholderType type; /// If [type] is [DartPlaceholderType.expression] and the expression could be @@ -231,6 +237,7 @@ class FoundDartPlaceholder { final ColumnType columnType; final String name; + DartPlaceholder astNode; FoundDartPlaceholder(this.type, this.columnType, this.name); @@ -254,4 +261,7 @@ class FoundDartPlaceholder { throw AssertionError('cant happen, all branches covered'); } + + @override + String get dartParameterName => name; } diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 9b4e426c..8f5595c8 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -9,13 +9,17 @@ import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; import 'package:sqlparser/sqlparser.dart'; -const highestAssignedIndexVar = '\$highestIndex'; +const highestAssignedIndexVar = '\$arrayStartIndex'; + +int _compareNodes(AstNode a, AstNode b) => + a.firstPosition.compareTo(b.firstPosition); /// Writes the handling code for a query. The code emitted will be a method that /// should be included in a generated database or dao class. class QueryWriter { final SqlQuery query; final Scope scope; + SqlSelectQuery get _select => query as SqlSelectQuery; UpdatingQuery get _update => query as UpdatingQuery; @@ -40,6 +44,10 @@ class QueryWriter { return 'expanded${v.dartParameterName}'; } + String _placeholderContextName(FoundDartPlaceholder placeholder) { + return 'generated${placeholder.name}'; + } + void write() { if (query is SqlSelectQuery) { final select = query as SqlSelectQuery; @@ -180,19 +188,19 @@ class QueryWriter { } void _writeParameters() { - final variableParams = query.variables.map((v) { - var dartType = dartTypeNames[v.type]; - if (v.isArray) { - dartType = 'List<$dartType>'; + final paramList = query.elements.map((e) { + if (e is FoundVariable) { + var dartType = dartTypeNames[e.type]; + if (e.isArray) { + dartType = 'List<$dartType>'; + } + return '$dartType ${e.dartParameterName}'; + } else if (e is FoundDartPlaceholder) { + return '${e.parameterType} ${e.name}'; } - return '$dartType ${v.dartParameterName}'; - }); - final placeholderParams = query.placeholders.map((p) { - return '${p.parameterType} ${p.name}'; - }); - - final paramList = variableParams.followedBy(placeholderParams).join(', '); + throw AssertionError('Unknown element (not variable of placeholder)'); + }).join(', '); _buffer.write(paramList); } @@ -200,10 +208,7 @@ class QueryWriter { /// assuming that for each parameter, a variable with the same name exists /// in the current scope. void _writeUseParameters() { - final parameters = query.variables - .map((v) => v.dartParameterName) - .followedBy(query.placeholders.map((p) => p.name)); - + final parameters = query.elements.map((e) => e.dartParameterName); _buffer.write(parameters.join(', ')); } @@ -218,42 +223,71 @@ class QueryWriter { // We use explicit indexes when expanding so that we don't have to expand the // "vars" variable twice. To do this, a local var called "$currentVarIndex" // keeps track of the highest variable number assigned. + // We can use the same mechanism for runtime Dart placeholders, where we + // generate a GenerationContext, write the placeholder and finally extract the + // variables void _writeExpandedDeclarations() { var indexCounterWasDeclared = false; var highestIndexBeforeArray = 0; - for (var variable in query.variables) { - if (variable.isArray) { - if (!indexCounterWasDeclared) { - // we only need the index counter when the query contains an array. - // add +1 because that's going to be the first index of the expanded - // array - final firstVal = highestIndexBeforeArray + 1; - _buffer.write('var $highestAssignedIndexVar = $firstVal;'); - indexCounterWasDeclared = true; + void _writeIndexCounter() { + // we only need the index counter when the query contains an expanded + // element. + // add +1 because that's going to be the first index of this element. + final firstVal = highestIndexBeforeArray + 1; + _buffer.write('var $highestAssignedIndexVar = $firstVal;'); + indexCounterWasDeclared = true; + } + + void _increaseIndexCounter(String by) { + _buffer..write('$highestAssignedIndexVar += ')..write(by)..write(';\n'); + } + + // query.elements are guaranteed to be sorted in the order in which they're + // going to have an effect when expanded. See TypeMapper.extractElements for + // the gory details. + for (var element in query.elements) { + if (element is FoundVariable) { + if (element.isArray) { + if (!indexCounterWasDeclared) { + _writeIndexCounter(); + } + + // final expandedvar1 = $expandVar(, ); + _buffer + ..write('final ') + ..write(_expandedName(element)) + ..write(' = ') + ..write(r'$expandVar(') + ..write(highestAssignedIndexVar) + ..write(', ') + ..write(element.dartParameterName) + ..write('.length);\n'); + + // increase highest index for the next expanded element + _increaseIndexCounter('${element.dartParameterName}.length'); } - // final expandedvar1 = $expandVar(, ); + if (!indexCounterWasDeclared) { + highestIndexBeforeArray = max(highestIndexBeforeArray, element.index); + } + } else if (element is FoundDartPlaceholder) { + if (!indexCounterWasDeclared) { + indexCounterWasDeclared = true; + } _buffer ..write('final ') - ..write(_expandedName(variable)) + ..write(_placeholderContextName(element)) ..write(' = ') - ..write(r'$expandVar(') - ..write(highestAssignedIndexVar) - ..write(', ') - ..write(variable.dartParameterName) - ..write('.length);\n'); + ..write(r'$write(') + ..write(element.dartParameterName) + ..write(');\n'); - // increase highest index for the next array - _buffer - ..write('$highestAssignedIndexVar += ') - ..write(variable.dartParameterName) - ..write('.length;'); - } - - if (!indexCounterWasDeclared) { - highestIndexBeforeArray = max(highestIndexBeforeArray, variable.index); + // similar to the case for expanded array variables, we need to + // increase the index + _increaseIndexCounter( + '${_placeholderContextName(element)}.amountOfVariables'); } } } @@ -283,29 +317,42 @@ class QueryWriter { /// been expanded. For instance, 'SELECT * FROM t WHERE x IN ?' will be turned /// into 'SELECT * FROM t WHERE x IN ($expandedVar1)'. String _queryCode() { - // sort variables by the order in which they appear - final vars = query.fromContext.root.allDescendants - .whereType() + // sort variables and placeholders by the order in which they appear + final toReplace = query.fromContext.root.allDescendants + .where((node) => node is Variable || node is DartPlaceholder) .toList() - ..sort((a, b) => a.firstPosition.compareTo(b.firstPosition)); + ..sort(_compareNodes); final buffer = StringBuffer("'"); var lastIndex = query.fromContext.root.firstPosition; - for (var sqlVar in vars) { - final moorVar = query.variables - .singleWhere((f) => f.variable.resolvedIndex == sqlVar.resolvedIndex); - if (!moorVar.isArray) continue; - + void replaceNode(AstNode node, String content) { // write everything that comes before this var into the buffer - final currentIndex = sqlVar.firstPosition; + final currentIndex = node.firstPosition; final queryPart = query.sql.substring(lastIndex, currentIndex); buffer.write(escapeForDart(queryPart)); - lastIndex = sqlVar.lastPosition; + lastIndex = node.lastPosition; - // write the ($expandedVar) par - buffer.write('(\$${_expandedName(moorVar)})'); + // write the replaced content + buffer.write(content); + } + + for (var rewriteTarget in toReplace) { + if (rewriteTarget is Variable) { + final moorVar = query.variables.singleWhere( + (f) => f.variable.resolvedIndex == rewriteTarget.resolvedIndex); + + if (moorVar.isArray) { + replaceNode(rewriteTarget, '(\$${_expandedName(moorVar)})'); + } + } else if (rewriteTarget is DartPlaceholder) { + final moorPlaceholder = + query.placeholders.singleWhere((p) => p.astNode == rewriteTarget); + + replaceNode(rewriteTarget, + '\${${_placeholderContextName(moorPlaceholder)}.sql}'); + } } // write the final part after the last variable, plus the ending ' diff --git a/moor_generator/test/analyzer/sql_queries/type_mapping_test.dart b/moor_generator/test/analyzer/sql_queries/type_mapping_test.dart new file mode 100644 index 00000000..59f35781 --- /dev/null +++ b/moor_generator/test/analyzer/sql_queries/type_mapping_test.dart @@ -0,0 +1,37 @@ +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; +import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:sqlparser/sqlparser.dart'; +import 'package:test/test.dart'; + +final _idColumn = TableColumn('id', const ResolvedType(type: BasicType.int)); +final _titleColumn = + TableColumn('title', const ResolvedType(type: BasicType.text)); +final table = Table(name: 'todos', resolvedColumns: [_idColumn, _titleColumn]); + +void main() { + final engine = SqlEngine()..registerTable(table); + final mapper = TypeMapper(); + + test('extracts variables and sorts them by index', () { + final result = engine.analyze( + 'SELECT * FROM todos WHERE title = ?2 OR id IN ? OR title = ?1'); + + final elements = mapper.extractElements(result).cast(); + + expect(elements.map((v) => v.index), [1, 2, 3]); + }); + + test('throws when an array with an explicit index is used', () { + final result = engine.analyze('SELECT 1 WHERE 1 IN ?1'); + + expect(() => mapper.extractElements(result), throwsArgumentError); + }); + + test( + 'throws when an explicitly index var with higher index appears after array', + () { + final result = engine.analyze('SELECT 1 WHERE 1 IN ? OR 2 = ?2'); + expect(() => mapper.extractElements(result), throwsArgumentError); + }, + ); +} diff --git a/sqlparser/lib/src/analysis/context.dart b/sqlparser/lib/src/analysis/context.dart index 379cd015..30ff09c6 100644 --- a/sqlparser/lib/src/analysis/context.dart +++ b/sqlparser/lib/src/analysis/context.dart @@ -29,4 +29,12 @@ class AnalysisContext { /// Obtains the result of any typeable component. See the information at /// [types] on important [Typeable]s. ResolveResult typeOf(Typeable t) => types.resolveOrInfer(t); + + /// Compares two [AstNode]s by their first position in the query. + static int compareNodesByOrder(AstNode first, AstNode second) { + if (first.first == null || second.first == null) { + return 0; // position not set. should we throw in that case? + } + return first.firstPosition.compareTo(second.firstPosition); + } } diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 5fbcacd7..bad78ebc 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -286,7 +286,8 @@ mixin CrudParser on ParserBase { // terms). if (terms.length == 1 && terms.single is DartOrderingTermPlaceholder) { final termPlaceholder = terms.single as DartOrderingTermPlaceholder; - return DartOrderByPlaceholder(name: termPlaceholder.name); + return DartOrderByPlaceholder(name: termPlaceholder.name) + ..setSpan(termPlaceholder.first, termPlaceholder.last); } return OrderBy(terms: terms); From 29c0cdaf3fbdf8d285b5c1fddbff06d5ff93a2f5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 16:48:51 +0200 Subject: [PATCH 087/117] Write integration test for Dart templates --- .../parsed_sql/moor_files_integration_test.dart | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/moor/test/parsed_sql/moor_files_integration_test.dart b/moor/test/parsed_sql/moor_files_integration_test.dart index 4ea88f04..a10a460b 100644 --- a/moor/test/parsed_sql/moor_files_integration_test.dart +++ b/moor/test/parsed_sql/moor_files_integration_test.dart @@ -54,4 +54,17 @@ void main() { await db.into(db.mytable).insert(const MytableCompanion()); verify(mock.runInsert('INSERT INTO mytable DEFAULT VALUES', [])); }); + + test('runs queries with arrays and Dart templates', () async { + final mock = MockExecutor(); + final db = CustomTablesDb(mock); + + await db.readMultiple(['a', 'b'], + OrderBy([OrderingTerm(expression: db.config.configKey)])).get(); + + verify(mock.runSelect( + 'SELECT * FROM config WHERE config_key IN (?1, ?2) ORDER BY config_key ASC', + ['a', 'b'], + )); + }); } From 54c3dbc4b88656e7611a1af21f16ab5cd6a076ef Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 17:48:17 +0200 Subject: [PATCH 088/117] Write variables created in a dart template --- moor/test/data/tables/custom_tables.g.dart | 14 +++------ moor/test/data/tables/todos.g.dart | 24 ++++---------- .../lib/src/writer/queries/query_writer.dart | 31 ++++++++++++------- 3 files changed, 30 insertions(+), 39 deletions(-) diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 137625e1..dea654b6 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -821,12 +821,8 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Selectable readConfig(String var1) { return customSelectQuery('SELECT * FROM config WHERE config_key = ?', - variables: [ - Variable.withString(var1), - ], - readsFrom: { - config - }).map(_rowToConfigData); + variables: [Variable.withString(var1)], + readsFrom: {config}).map(_rowToConfigData); } Selectable readMultiple(List var1, OrderBy clause) { @@ -839,6 +835,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { 'SELECT * FROM config WHERE config_key IN ($expandedvar1) ORDER BY ${generatedclause.sql}', variables: [ for (var $ in var1) Variable.withString($), + ...generatedclause.introducedVariables ], readsFrom: { config @@ -848,10 +845,7 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { Future writeConfig(String key, String value) { return customInsert( 'REPLACE INTO config VALUES (:key, :value)', - variables: [ - Variable.withString(key), - Variable.withString(value), - ], + variables: [Variable.withString(key), Variable.withString(value)], updates: {config}, ); } diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index da9033ff..60bf6d0f 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1333,9 +1333,7 @@ abstract class _$TodoDb extends GeneratedDatabase { Future deleteTodoById(int var1) { return customUpdate( 'DELETE FROM todos WHERE id = ?', - variables: [ - Variable.withInt(var1), - ], + variables: [Variable.withInt(var1)], updates: {todosTable}, ); } @@ -1359,7 +1357,7 @@ abstract class _$TodoDb extends GeneratedDatabase { variables: [ Variable.withString(var1), Variable.withString(var2), - for (var $ in var3) Variable.withInt($), + for (var $ in var3) Variable.withInt($) ], readsFrom: { todosTable @@ -1378,12 +1376,8 @@ abstract class _$TodoDb extends GeneratedDatabase { Selectable searchQuery(int id) { return customSelectQuery( 'SELECT * FROM todos WHERE CASE WHEN -1 = :id THEN 1 ELSE id = :id END', - variables: [ - Variable.withInt(id), - ], - readsFrom: { - todosTable - }).map(_rowToTodoEntry); + variables: [Variable.withInt(id)], + readsFrom: {todosTable}).map(_rowToTodoEntry); } Future> search(int id) { @@ -1474,14 +1468,8 @@ mixin _$SomeDaoMixin on DatabaseAccessor { Selectable todosForUserQuery(int user) { return customSelectQuery( 'SELECT t.* FROM todos t INNER JOIN shared_todos st ON st.todo = t.id INNER JOIN users u ON u.id = st.user WHERE u.id = :user', - variables: [ - Variable.withInt(user), - ], - readsFrom: { - todosTable, - sharedTodos, - users - }).map(_rowToTodoEntry); + variables: [Variable.withInt(user)], + readsFrom: {todosTable, sharedTodos, users}).map(_rowToTodoEntry); } Future> todosForUser(int user) { diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 8f5595c8..0b71126d 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -295,19 +295,28 @@ class QueryWriter { void _writeVariables() { _buffer..write('variables: ['); - for (var variable in query.variables) { - // for a regular variable: Variable.withInt(x), - // for a list of vars: for (var $ in vars) Variable.withInt($), - final constructor = createVariable[variable.type]; - final name = variable.dartParameterName; - - if (variable.isArray) { - _buffer.write('for (var \$ in $name) $constructor(\$)'); - } else { - _buffer.write('$constructor($name)'); + var first = true; + for (var element in query.elements) { + if (!first) { + _buffer.write(', '); } + first = false; - _buffer.write(','); + if (element is FoundVariable) { + // for a regular variable: Variable.withInt(x), + // for a list of vars: for (var $ in vars) Variable.withInt($), + final constructor = createVariable[element.type]; + final name = element.dartParameterName; + + if (element.isArray) { + _buffer.write('for (var \$ in $name) $constructor(\$)'); + } else { + _buffer.write('$constructor($name)'); + } + } else if (element is FoundDartPlaceholder) { + _buffer.write( + '...${_placeholderContextName(element)}.introducedVariables'); + } } _buffer..write(']'); From e89f9227abbf7df9476b8c56a8cbf1d2c732789e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 18:53:27 +0200 Subject: [PATCH 089/117] Don't write ResultSet classes for single-column queries Fixes #148 --- .../tests/lib/database/database.g.dart | 149 ++++-------------- .../tests/lib/suite/custom_objects.dart | 8 +- .../tests/lib/suite/migrations.dart | 8 +- .../tests/lib/suite/transactions.dart | 8 +- moor/test/data/tables/todos.g.dart | 28 +--- moor_generator/lib/src/model/sql_query.dart | 22 +++ .../lib/src/writer/queries/query_writer.dart | 51 +++--- .../src/writer/queries/result_set_writer.dart | 11 +- 8 files changed, 110 insertions(+), 175 deletions(-) diff --git a/extras/integration_tests/tests/lib/database/database.g.dart b/extras/integration_tests/tests/lib/database/database.g.dart index 51eba060..8e5c09ed 100644 --- a/extras/integration_tests/tests/lib/database/database.g.dart +++ b/extras/integration_tests/tests/lib/database/database.g.dart @@ -552,165 +552,84 @@ abstract class _$Database extends GeneratedDatabase { ); } - Selectable mostPopularUsersQuery( - int amount, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelectQuery( + Selectable mostPopularUsersQuery(int amount) { + return customSelectQuery( 'SELECT * FROM users u ORDER BY (SELECT COUNT(*) FROM friendships WHERE first_user = u.id OR second_user = u.id) DESC LIMIT :amount', - variables: [ - Variable.withInt(amount), - ], - readsFrom: { - users, - friendships - }).map(_rowToUser); + variables: [Variable.withInt(amount)], + readsFrom: {users, friendships}).map(_rowToUser); } - Future> mostPopularUsers( - int amount, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return mostPopularUsersQuery(amount, operateOn: operateOn).get(); + Future> mostPopularUsers(int amount) { + return mostPopularUsersQuery(amount).get(); } Stream> watchMostPopularUsers(int amount) { return mostPopularUsersQuery(amount).watch(); } - AmountOfGoodFriendsResult _rowToAmountOfGoodFriendsResult(QueryRow row) { - return AmountOfGoodFriendsResult( - count: row.readInt('COUNT(*)'), - ); - } - - Selectable amountOfGoodFriendsQuery( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelectQuery( + Selectable amountOfGoodFriendsQuery(int user) { + return customSelectQuery( 'SELECT COUNT(*) FROM friendships f WHERE f.really_good_friends AND (f.first_user = :user OR f.second_user = :user)', variables: [ - Variable.withInt(user), + Variable.withInt(user) ], readsFrom: { friendships - }).map(_rowToAmountOfGoodFriendsResult); + }).map((QueryRow row) => row.readInt('COUNT(*)')); } - Future> amountOfGoodFriends( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return amountOfGoodFriendsQuery(user, operateOn: operateOn).get(); + Future> amountOfGoodFriends(int user) { + return amountOfGoodFriendsQuery(user).get(); } - Stream> watchAmountOfGoodFriends(int user) { + Stream> watchAmountOfGoodFriends(int user) { return amountOfGoodFriendsQuery(user).watch(); } - Selectable friendsOfQuery( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelectQuery( + Selectable friendsOfQuery(int user) { + return customSelectQuery( 'SELECT u.* FROM friendships f\n INNER JOIN users u ON u.id IN (f.first_user, f.second_user) AND\n u.id != :user\n WHERE (f.first_user = :user OR f.second_user = :user)', - variables: [ - Variable.withInt(user), - ], - readsFrom: { - friendships, - users - }).map(_rowToUser); + variables: [Variable.withInt(user)], + readsFrom: {friendships, users}).map(_rowToUser); } - Future> friendsOf( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return friendsOfQuery(user, operateOn: operateOn).get(); + Future> friendsOf(int user) { + return friendsOfQuery(user).get(); } Stream> watchFriendsOf(int user) { return friendsOfQuery(user).watch(); } - UserCountResult _rowToUserCountResult(QueryRow row) { - return UserCountResult( - cOUNTid: row.readInt('COUNT(id)'), - ); + Selectable userCountQuery() { + return customSelectQuery('SELECT COUNT(id) FROM users', + variables: [], + readsFrom: {users}).map((QueryRow row) => row.readInt('COUNT(id)')); } - Selectable userCountQuery( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelectQuery('SELECT COUNT(id) FROM users', - variables: [], readsFrom: {users}).map(_rowToUserCountResult); + Future> userCount() { + return userCountQuery().get(); } - Future> userCount( - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return userCountQuery(operateOn: operateOn).get(); - } - - Stream> watchUserCount() { + Stream> watchUserCount() { return userCountQuery().watch(); } - SettingsForResult _rowToSettingsForResult(QueryRow row) { - return SettingsForResult( - preferences: - $UsersTable.$converter0.mapToDart(row.readString('preferences')), - ); + Selectable settingsForQuery(int user) { + return customSelectQuery('SELECT preferences FROM users WHERE id = :user', + variables: [Variable.withInt(user)], readsFrom: {users}) + .map((QueryRow row) => + $UsersTable.$converter0.mapToDart(row.readString('preferences'))); } - Selectable settingsForQuery( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return (operateOn ?? this).customSelectQuery( - 'SELECT preferences FROM users WHERE id = :user', - variables: [ - Variable.withInt(user), - ], - readsFrom: { - users - }).map(_rowToSettingsForResult); + Future> settingsFor(int user) { + return settingsForQuery(user).get(); } - Future> settingsFor( - int user, - {@Deprecated('No longer needed with Moor 1.6 - see the changelog for details') - QueryEngine operateOn}) { - return settingsForQuery(user, operateOn: operateOn).get(); - } - - Stream> watchSettingsFor(int user) { + Stream> watchSettingsFor(int user) { return settingsForQuery(user).watch(); } @override List get allTables => [users, friendships]; } - -class AmountOfGoodFriendsResult { - final int count; - AmountOfGoodFriendsResult({ - this.count, - }); -} - -class UserCountResult { - final int cOUNTid; - UserCountResult({ - this.cOUNTid, - }); -} - -class SettingsForResult { - final Preferences preferences; - SettingsForResult({ - this.preferences, - }); -} diff --git a/extras/integration_tests/tests/lib/suite/custom_objects.dart b/extras/integration_tests/tests/lib/suite/custom_objects.dart index 93ef5f78..d5d57fdd 100644 --- a/extras/integration_tests/tests/lib/suite/custom_objects.dart +++ b/extras/integration_tests/tests/lib/suite/custom_objects.dart @@ -7,13 +7,13 @@ void customObjectTests(TestExecutor executor) { test('custom objects', () async { final db = Database(executor.createExecutor()); - var preferences = await db.settingsFor(1); - expect(preferences.single.preferences, isNull); + var preferences = await db.settingsForQuery(1).getSingle(); + expect(preferences, isNull); await db.updateSettings(1, Preferences(true)); - preferences = await db.settingsFor(1); + preferences = await db.settingsForQuery(1).getSingle(); - expect(preferences.single.preferences.receiveEmails, true); + expect(preferences.receiveEmails, true); await db.close(); }); diff --git a/extras/integration_tests/tests/lib/suite/migrations.dart b/extras/integration_tests/tests/lib/suite/migrations.dart index af41b2b5..ba6afa08 100644 --- a/extras/integration_tests/tests/lib/suite/migrations.dart +++ b/extras/integration_tests/tests/lib/suite/migrations.dart @@ -9,8 +9,8 @@ void migrationTests(TestExecutor executor) { final database = Database(executor.createExecutor(), schemaVersion: 1); // we write 3 users when the database is created - final count = await database.userCount(); - expect(count.single.cOUNTid, 3); + final count = await database.userCountQuery().getSingle(); + expect(count, 3); await database.close(); }); @@ -23,8 +23,8 @@ void migrationTests(TestExecutor executor) { database = Database(executor.createExecutor(), schemaVersion: 2); // the 3 initial users plus People.florian - final count = await database.userCount(); - expect(count.single.cOUNTid, 4); + final count = await database.userCountQuery().getSingle(); + expect(count, 4); await database.close(); }); diff --git a/extras/integration_tests/tests/lib/suite/transactions.dart b/extras/integration_tests/tests/lib/suite/transactions.dart index 05b6f3b9..91664e02 100644 --- a/extras/integration_tests/tests/lib/suite/transactions.dart +++ b/extras/integration_tests/tests/lib/suite/transactions.dart @@ -19,10 +19,10 @@ void transactionTests(TestExecutor executor) { }); final countResult = await db.userCount(); - expect(countResult.single.cOUNTid, 4); + expect(countResult.single, 4); final friendsResult = await db.amountOfGoodFriends(People.dashId); - expect(friendsResult.single.count, 1); + expect(friendsResult.single, 1); await db.close(); }); @@ -45,10 +45,10 @@ void transactionTests(TestExecutor executor) { } on Exception catch (_) {} final countResult = await db.userCount(); - expect(countResult.single.cOUNTid, 3); // only the default folks + expect(countResult.single, 3); // only the default folks final friendsResult = await db.amountOfGoodFriends(People.dashId); - expect(friendsResult.single.count, 0); // no friendship was inserted + expect(friendsResult.single, 0); // no friendship was inserted await db.close(); }); diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 60bf6d0f..53082979 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -1388,25 +1388,20 @@ abstract class _$TodoDb extends GeneratedDatabase { return searchQuery(id).watch(); } - FindCustomResult _rowToFindCustomResult(QueryRow row) { - return FindCustomResult( - custom: - $TableWithoutPKTable.$converter0.mapToDart(row.readString('custom')), - ); - } - - Selectable findCustomQuery() { + Selectable findCustomQuery() { return customSelectQuery( - 'SELECT custom FROM table_without_p_k WHERE some_float < 10', - variables: [], - readsFrom: {tableWithoutPK}).map(_rowToFindCustomResult); + 'SELECT custom FROM table_without_p_k WHERE some_float < 10', + variables: [], + readsFrom: {tableWithoutPK}) + .map((QueryRow row) => $TableWithoutPKTable.$converter0 + .mapToDart(row.readString('custom'))); } - Future> findCustom() { + Future> findCustom() { return findCustomQuery().get(); } - Stream> watchFindCustom() { + Stream> watchFindCustom() { return findCustomQuery().watch(); } @@ -1440,13 +1435,6 @@ class AllTodosWithCategoryResult { }); } -class FindCustomResult { - final MyCustomObject custom; - FindCustomResult({ - this.custom, - }); -} - // ************************************************************************** // DaoGenerator // ************************************************************************** diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index e68317bc..b5cd4a68 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -101,6 +101,11 @@ class SqlSelectQuery extends SqlQuery { if (resultSet.matchingTable != null) { return resultSet.matchingTable.dartTypeName; } + + if (resultSet.singleColumn) { + return resultSet.columns.single.dartType; + } + return '${ReCase(name).pascalCase}Result'; } @@ -129,6 +134,14 @@ class InferredResultSet { InferredResultSet(this.matchingTable, this.columns); + /// Whether a new class needs to be written to store the result of this query. + /// We don't need to do that for queries which return an existing table model + /// or if they only return exactly one column. + bool get needsOwnClass => matchingTable == null && !singleColumn; + + /// Whether this query returns a single column. + bool get singleColumn => columns.length == 1; + void forceDartNames(Map names) { _dartNames ..clear() @@ -173,6 +186,15 @@ class ResultColumn { final UsedTypeConverter converter; ResultColumn(this.name, this.type, this.nullable, {this.converter}); + + /// The dart type that can store a result of this column. + String get dartType { + if (converter != null) { + return converter.mappedType.displayName; + } else { + return dartTypeNames[type]; + } + } } /// Something in the query that needs special attention when generating code, diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 0b71126d..f2fec6c5 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -7,7 +7,7 @@ import 'package:moor_generator/src/utils/string_escaper.dart'; import 'package:moor_generator/src/writer/queries/result_set_writer.dart'; import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; -import 'package:sqlparser/sqlparser.dart'; +import 'package:sqlparser/sqlparser.dart' hide ResultColumn; const highestAssignedIndexVar = '\$arrayStartIndex'; @@ -51,8 +51,7 @@ class QueryWriter { void write() { if (query is SqlSelectQuery) { final select = query as SqlSelectQuery; - if (select.resultSet.matchingTable == null) { - // query needs its own result set - write that now + if (select.resultSet.needsOwnClass) { final buffer = scope.findScopeOfLevel(DartScope.library).leaf(); ResultSetWriter(select).write(buffer); } @@ -63,7 +62,10 @@ class QueryWriter { } void _writeSelect() { - _writeMapping(); + if (!_select.resultSet.singleColumn) { + _writeMapping(); + } + _writeSelectStatementCreator(); if (!query.declaredInMoorFile) { @@ -97,19 +99,7 @@ class QueryWriter { for (var column in _select.resultSet.columns) { final fieldName = _select.resultSet.dartNameFor(column); - final readMethod = readFromMethods[column.type]; - - var code = "row.$readMethod('${column.name}')"; - - if (column.converter != null) { - final converter = column.converter; - final infoName = converter.table.tableInfoName; - final field = '$infoName.${converter.fieldName}'; - - code = '$field.mapToDart($code)'; - } - - _buffer.write('$fieldName: $code,'); + _buffer.write('$fieldName: ${_readingCode(column)},'); } _buffer.write(');\n}\n'); @@ -117,6 +107,24 @@ class QueryWriter { } } + /// Returns Dart code that, given a variable of type `QueryRow` named `row` + /// in the same scope, reads the [column] from that row and brings it into a + /// suitable type. + String _readingCode(ResultColumn column) { + final readMethod = readFromMethods[column.type]; + + var code = "row.$readMethod('${column.name}')"; + + if (column.converter != null) { + final converter = column.converter; + final infoName = converter.table.tableInfoName; + final field = '$infoName.${converter.fieldName}'; + + code = '$field.mapToDart($code)'; + } + return code; + } + /// Writes a method returning a `Selectable`, where `T` is the return type /// of the custom query. void _writeSelectStatementCreator() { @@ -134,7 +142,14 @@ class QueryWriter { _writeReadsFrom(); _buffer.write(').map('); - _buffer.write(_nameOfMappingMethod()); + // for queries that only return one row, it makes more sense to inline the + // mapping code with a lambda + if (_select.resultSet.singleColumn) { + final column = _select.resultSet.columns.single; + _buffer.write('(QueryRow row) => ${_readingCode(column)}'); + } else { + _buffer.write(_nameOfMappingMethod()); + } _buffer.write(');\n}\n'); } diff --git a/moor_generator/lib/src/writer/queries/result_set_writer.dart b/moor_generator/lib/src/writer/queries/result_set_writer.dart index 88ea7d63..3399a6b6 100644 --- a/moor_generator/lib/src/writer/queries/result_set_writer.dart +++ b/moor_generator/lib/src/writer/queries/result_set_writer.dart @@ -1,4 +1,3 @@ -import 'package:moor_generator/src/model/specified_column.dart'; import 'package:moor_generator/src/model/sql_query.dart'; /// Writes a class holding the result of an sql query into Dart. @@ -14,7 +13,7 @@ class ResultSetWriter { // write fields for (var column in query.resultSet.columns) { final name = query.resultSet.dartNameFor(column); - final runtimeType = _getRuntimeType(column); + final runtimeType = column.dartType; into.write('final $runtimeType $name\n;'); } @@ -25,12 +24,4 @@ class ResultSetWriter { } into.write('});\n}\n'); } - - String _getRuntimeType(ResultColumn column) { - if (column.converter != null) { - return column.converter.mappedType.displayName; - } else { - return dartTypeNames[column.type]; - } - } } From 928c9832b5c88ffd83dc370f03f0a4520eee1fb8 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 20:54:13 +0200 Subject: [PATCH 090/117] Move ffi bindings into new moor_ffi package --- moor/CHANGELOG.md | 8 +- moor/lib/moor_vm.dart | 15 --- moor/test/vm/integration_test.dart | 105 ------------------ moor_ffi/.gitignore | 12 ++ moor_ffi/CHANGELOG.md | 3 + moor_ffi/LICENSE | 24 ++++ moor_ffi/README.md | 14 +++ moor_ffi/android/.gitignore | 12 ++ moor_ffi/android/build.gradle | 67 +++++++++++ moor_ffi/android/cpp/CMakeLists.txt | 4 + moor_ffi/android/gradle.properties | 2 + .../gradle/wrapper/gradle-wrapper.properties | 5 + moor_ffi/android/settings.gradle | 1 + moor_ffi/ios/.gitignore | 37 ++++++ moor_ffi/ios/Assets/.gitkeep | 0 moor_ffi/ios/Classes/MoorFfiPlugin.h | 4 + moor_ffi/ios/Classes/MoorFfiPlugin.m | 8 ++ moor_ffi/ios/Classes/SwiftMoorFfiPlugin.swift | 14 +++ moor_ffi/ios/moor_ffi.podspec | 21 ++++ moor_ffi/lib/moor_ffi.dart | 7 ++ .../vm => moor_ffi/lib/src}/api/database.dart | 10 +- .../vm => moor_ffi/lib/src}/api/errors.dart | 0 .../lib/src}/api/prepared_statement.dart | 0 .../vm => moor_ffi/lib/src}/api/result.dart | 0 .../lib/src}/bindings/bindings.dart | 0 .../lib/src}/bindings/constants.dart | 0 .../lib/src}/bindings/signatures.dart | 0 .../lib/src}/bindings/types.dart | 0 .../src/vm => moor_ffi/lib/src}/ffi/blob.dart | 3 +- .../lib/src}/ffi/open_platform_specific.dart | 0 .../vm => moor_ffi/lib/src}/ffi/utils.dart | 0 .../vm => moor_ffi/lib/src}/vm_database.dart | 2 +- moor_ffi/pubspec.yaml | 25 +++++ 33 files changed, 275 insertions(+), 128 deletions(-) delete mode 100644 moor/lib/moor_vm.dart delete mode 100644 moor/test/vm/integration_test.dart create mode 100644 moor_ffi/.gitignore create mode 100644 moor_ffi/CHANGELOG.md create mode 100644 moor_ffi/LICENSE create mode 100644 moor_ffi/README.md create mode 100644 moor_ffi/android/.gitignore create mode 100644 moor_ffi/android/build.gradle create mode 100644 moor_ffi/android/cpp/CMakeLists.txt create mode 100644 moor_ffi/android/gradle.properties create mode 100644 moor_ffi/android/gradle/wrapper/gradle-wrapper.properties create mode 100644 moor_ffi/android/settings.gradle create mode 100644 moor_ffi/ios/.gitignore create mode 100644 moor_ffi/ios/Assets/.gitkeep create mode 100644 moor_ffi/ios/Classes/MoorFfiPlugin.h create mode 100644 moor_ffi/ios/Classes/MoorFfiPlugin.m create mode 100644 moor_ffi/ios/Classes/SwiftMoorFfiPlugin.swift create mode 100644 moor_ffi/ios/moor_ffi.podspec create mode 100644 moor_ffi/lib/moor_ffi.dart rename {moor/lib/src/vm => moor_ffi/lib/src}/api/database.dart (94%) rename {moor/lib/src/vm => moor_ffi/lib/src}/api/errors.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/api/prepared_statement.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/api/result.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/bindings/bindings.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/bindings/constants.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/bindings/signatures.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/bindings/types.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/ffi/blob.dart (93%) rename {moor/lib/src/vm => moor_ffi/lib/src}/ffi/open_platform_specific.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/ffi/utils.dart (100%) rename {moor/lib/src/vm => moor_ffi/lib/src}/vm_database.dart (98%) create mode 100644 moor_ffi/pubspec.yaml diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 26b352b5..2accea65 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -9,6 +9,10 @@ TODO: Properly describe these additions when they're finalized: - Analyzer plugin for Dart Code - `ffi` libraries +### Minor changes +- a `Constant` can now be written to SQL, it used to throw before. This is useful + if you need default values for strings columns. + ### Breaking changes - __THIS LIKELY AFFECTS YOUR APP:__ Removed the `transaction` parameter for callbacks in transactions and `beforeOpen` callbacks. So, instead of writing @@ -28,7 +32,9 @@ TODO: Properly describe these additions when they're finalized: your database instead of a transaction objects. They will be delegated automatically. On a similar note, we also removed the `operateOn` parameter from compiled queries. - +- Compiled queries that return only a single column (e.g. `SELECT COUNT(*) FROM users`) + will just return their value (in this case, an `int`) now. Moor no longer generates a + new class in that case. - Removed `MigrationStrategy.onFinished`. Use `beforeOpen` instead. - Compiled sql queries starting with an underscore will now generate private match queries. Previously, the query `_allUsers` would generate a `watchAllUsers` method, that has been diff --git a/moor/lib/moor_vm.dart b/moor/lib/moor_vm.dart deleted file mode 100644 index 314ec4f5..00000000 --- a/moor/lib/moor_vm.dart +++ /dev/null @@ -1,15 +0,0 @@ -/// A version of moor that runs on the Dart VM by integrating sqlite3 with -/// ffi. -@experimental -library moor_vm; - -import 'dart:async'; -import 'dart:io'; - -import 'package:meta/meta.dart'; -import 'backends.dart'; -import 'moor.dart'; - -import 'src/vm/api/database.dart'; - -part 'src/vm/vm_database.dart'; diff --git a/moor/test/vm/integration_test.dart b/moor/test/vm/integration_test.dart deleted file mode 100644 index f42d7131..00000000 --- a/moor/test/vm/integration_test.dart +++ /dev/null @@ -1,105 +0,0 @@ -import 'dart:async'; - -import 'package:moor/moor.dart'; -import 'package:pedantic/pedantic.dart'; -import 'package:test_api/test_api.dart'; -import 'package:moor/moor_vm.dart'; - -import '../data/tables/todos.dart'; - -TodoDb db; - -void main() { - test('CRUD integration test', () async { - db = TodoDb(VMDatabase.memory(logStatements: false)); - - // write some dummy data - await insertCategory(); - await insertUser(); - await insertTodos(); - - await db.into(db.sharedTodos).insert(SharedTodo(todo: 2, user: 1)); - - // test select statements - final forUser = (await db.someDao.todosForUser(1)).single; - expect(forUser.title, 'Another entry'); - - // test delete statements - await db.deleteTodoById(2); - final queryAgain = await db.someDao.todosForUser(1); - expect(queryAgain, isEmpty); - - // test update statements - await (db.update(db.todosTable)..where((t) => t.id.equals(1))) - .write(const TodosTableCompanion(content: Value('Updated content'))); - final readUpdated = await db.select(db.todosTable).getSingle(); - expect(readUpdated.content, 'Updated content'); - }); - - test('Transactions test', () async { - db = TodoDb(VMDatabase.memory(logStatements: false)); - - final completedOperations = StreamController(); - - unawaited(db.transaction((_) async { - await insertCategory(); - completedOperations.add('transaction'); - await pumpEventQueue(); - })); - - unawaited(insertUser().then((_) { - completedOperations.add('regular'); - })); - - await expectLater( - completedOperations.stream, emitsInOrder(['transaction', 'regular'])); - - // call .getSingle to verify both rows have been written - await db.select(db.users).getSingle(); - await db.select(db.categories).getSingle(); - }); -} - -Future insertCategory() async { - final forInsert = const CategoriesCompanion(description: Value('Work')); - final row = Category(id: 1, description: 'Work'); - - final id = await db.into(db.categories).insert(forInsert); - expect(id, equals(1)); - - final loaded = await db.select(db.categories).getSingle(); - expect(loaded, equals(row)); -} - -Future insertUser() async { - final profilePic = Uint8List.fromList([1, 2, 3, 4, 5, 6]); - final forInsert = UsersCompanion( - name: const Value('Dashy McDashface'), - isAwesome: const Value(true), - profilePicture: Value(profilePic), - ); - - final id = await db.into(db.users).insert(forInsert); - expect(id, equals(1)); - - final user = await db.select(db.users).getSingle(); - expect(user.id, equals(1)); - expect(user.name, equals('Dashy McDashface')); - expect(user.isAwesome, isTrue); - expect(user.profilePicture, profilePic); -} - -Future insertTodos() async { - await db.into(db.todosTable).insertAll([ - TodosTableCompanion( - title: const Value('A first entry'), - content: const Value('Some content I guess'), - targetDate: Value(DateTime(2019)), - ), - const TodosTableCompanion( - title: Value('Another entry'), - content: Value('this is a really creative test case'), - category: Value(1), // "Work" - ), - ]); -} diff --git a/moor_ffi/.gitignore b/moor_ffi/.gitignore new file mode 100644 index 00000000..e38f4862 --- /dev/null +++ b/moor_ffi/.gitignore @@ -0,0 +1,12 @@ +.DS_Store +.dart_tool/ + +.packages +.pub/ + +build/ + +pubspec.lock + +# todo determine whether metadata should be added to gitignore (the file says it shouldn't, but does this break)? +.metadata \ No newline at end of file diff --git a/moor_ffi/CHANGELOG.md b/moor_ffi/CHANGELOG.md new file mode 100644 index 00000000..41cc7d81 --- /dev/null +++ b/moor_ffi/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +* TODO: Describe initial release. diff --git a/moor_ffi/LICENSE b/moor_ffi/LICENSE new file mode 100644 index 00000000..c75eb6c8 --- /dev/null +++ b/moor_ffi/LICENSE @@ -0,0 +1,24 @@ +MIT License + +Copyright (c) 2019 Simon Binder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +This project also bundles sqlite, which is in the Public Domain. +See https://www.sqlite.org/copyright.html \ No newline at end of file diff --git a/moor_ffi/README.md b/moor_ffi/README.md new file mode 100644 index 00000000..80295ad6 --- /dev/null +++ b/moor_ffi/README.md @@ -0,0 +1,14 @@ +# moor_ffi + +Moor backend that uses the new `dart:ffi` apis. + +## Supported platforms +At the moment, this plugin supports Android natively. However, it's also going to run on all +platforms that expose `sqlite3` as a shared native library (macOS and virtually all Linux +distros). + +## Notes + +Using `flutter run` or `flutter build` when this library is imported is going to take very long for +the first time. The reason is that we compile sqlite. Subsequent builds should take an acceptable +time execute. \ No newline at end of file diff --git a/moor_ffi/android/.gitignore b/moor_ffi/android/.gitignore new file mode 100644 index 00000000..4a38248d --- /dev/null +++ b/moor_ffi/android/.gitignore @@ -0,0 +1,12 @@ +*.iml +.gradle +/local.properties +/.idea/workspace.xml +/.idea/libraries +.DS_Store +/build +/captures + +.externalNativeBuild/ +cpp/sqlite* +sqlite_*.zip \ No newline at end of file diff --git a/moor_ffi/android/build.gradle b/moor_ffi/android/build.gradle new file mode 100644 index 00000000..c37ae8bb --- /dev/null +++ b/moor_ffi/android/build.gradle @@ -0,0 +1,67 @@ +buildscript { + repositories { + google() + jcenter() + } + + dependencies { + classpath 'com.android.tools.build:gradle:3.5.0' + } +} + +plugins { + id "de.undercouch.download" version "4.0.0" +} + +group 'com.example.moor_ffi' +version '1.0' + + +rootProject.allprojects { + repositories { + google() + jcenter() + } +} + +apply plugin: 'com.android.library' + +android { + compileSdkVersion 28 + + defaultConfig { + minSdkVersion 16 + + externalNativeBuild { + cmake { + cppFlags "" + } + } + } + + externalNativeBuild { + cmake { + path "cpp/CMakeLists.txt" + } + } + + lintOptions { + disable 'InvalidPackage' + } +} + +task downloadSqlite(type: Download) { + src 'https://sqlite.org/2019/sqlite-amalgamation-3290000.zip' + dest 'sqlite_3290000.zip' + overwrite false +} + +task extractSqlite(dependsOn: downloadSqlite, type: Copy) { + from zipTree(downloadSqlite.dest).matching { + include '*/sqlite3.c' + eachFile { it.setPath(it.getName()) } // Don't use top-level folder in zip + } + into 'cpp' +} + +preBuild.dependsOn extractSqlite \ No newline at end of file diff --git a/moor_ffi/android/cpp/CMakeLists.txt b/moor_ffi/android/cpp/CMakeLists.txt new file mode 100644 index 00000000..4e6a46d8 --- /dev/null +++ b/moor_ffi/android/cpp/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.4.1) +project(sqlite3) + +add_library(sqlite3 SHARED sqlite3.c) diff --git a/moor_ffi/android/gradle.properties b/moor_ffi/android/gradle.properties new file mode 100644 index 00000000..2bd6f4fd --- /dev/null +++ b/moor_ffi/android/gradle.properties @@ -0,0 +1,2 @@ +org.gradle.jvmargs=-Xmx1536M + diff --git a/moor_ffi/android/gradle/wrapper/gradle-wrapper.properties b/moor_ffi/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..01a286e9 --- /dev/null +++ b/moor_ffi/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip diff --git a/moor_ffi/android/settings.gradle b/moor_ffi/android/settings.gradle new file mode 100644 index 00000000..2e83ea9c --- /dev/null +++ b/moor_ffi/android/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'moor_ffi' diff --git a/moor_ffi/ios/.gitignore b/moor_ffi/ios/.gitignore new file mode 100644 index 00000000..aa479fd3 --- /dev/null +++ b/moor_ffi/ios/.gitignore @@ -0,0 +1,37 @@ +.idea/ +.vagrant/ +.sconsign.dblite +.svn/ + +.DS_Store +*.swp +profile + +DerivedData/ +build/ +GeneratedPluginRegistrant.h +GeneratedPluginRegistrant.m + +.generated/ + +*.pbxuser +*.mode1v3 +*.mode2v3 +*.perspectivev3 + +!default.pbxuser +!default.mode1v3 +!default.mode2v3 +!default.perspectivev3 + +xcuserdata + +*.moved-aside + +*.pyc +*sync/ +Icon? +.tags* + +/Flutter/Generated.xcconfig +/Flutter/flutter_export_environment.sh \ No newline at end of file diff --git a/moor_ffi/ios/Assets/.gitkeep b/moor_ffi/ios/Assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/moor_ffi/ios/Classes/MoorFfiPlugin.h b/moor_ffi/ios/Classes/MoorFfiPlugin.h new file mode 100644 index 00000000..a47cb500 --- /dev/null +++ b/moor_ffi/ios/Classes/MoorFfiPlugin.h @@ -0,0 +1,4 @@ +#import + +@interface MoorFfiPlugin : NSObject +@end diff --git a/moor_ffi/ios/Classes/MoorFfiPlugin.m b/moor_ffi/ios/Classes/MoorFfiPlugin.m new file mode 100644 index 00000000..9745116d --- /dev/null +++ b/moor_ffi/ios/Classes/MoorFfiPlugin.m @@ -0,0 +1,8 @@ +#import "MoorFfiPlugin.h" +#import + +@implementation MoorFfiPlugin ++ (void)registerWithRegistrar:(NSObject*)registrar { + [SwiftMoorFfiPlugin registerWithRegistrar:registrar]; +} +@end diff --git a/moor_ffi/ios/Classes/SwiftMoorFfiPlugin.swift b/moor_ffi/ios/Classes/SwiftMoorFfiPlugin.swift new file mode 100644 index 00000000..dac1b09f --- /dev/null +++ b/moor_ffi/ios/Classes/SwiftMoorFfiPlugin.swift @@ -0,0 +1,14 @@ +import Flutter +import UIKit + +public class SwiftMoorFfiPlugin: NSObject, FlutterPlugin { + public static func register(with registrar: FlutterPluginRegistrar) { + let channel = FlutterMethodChannel(name: "moor_ffi", binaryMessenger: registrar.messenger()) + let instance = SwiftMoorFfiPlugin() + registrar.addMethodCallDelegate(instance, channel: channel) + } + + public func handle(_ call: FlutterMethodCall, result: @escaping FlutterResult) { + result("iOS " + UIDevice.current.systemVersion) + } +} diff --git a/moor_ffi/ios/moor_ffi.podspec b/moor_ffi/ios/moor_ffi.podspec new file mode 100644 index 00000000..cd4ea5e0 --- /dev/null +++ b/moor_ffi/ios/moor_ffi.podspec @@ -0,0 +1,21 @@ +# +# To learn more about a Podspec see http://guides.cocoapods.org/syntax/podspec.html +# +Pod::Spec.new do |s| + s.name = 'moor_ffi' + s.version = '0.0.1' + s.summary = 'A new flutter plugin project.' + s.description = <<-DESC +A new flutter plugin project. + DESC + s.homepage = 'http://example.com' + s.license = { :file => '../LICENSE' } + s.author = { 'Your Company' => 'email@example.com' } + s.source = { :path => '.' } + s.source_files = 'Classes/**/*' + s.public_header_files = 'Classes/**/*.h' + s.dependency 'Flutter' + + s.ios.deployment_target = '8.0' +end + diff --git a/moor_ffi/lib/moor_ffi.dart b/moor_ffi/lib/moor_ffi.dart new file mode 100644 index 00000000..c2d05425 --- /dev/null +++ b/moor_ffi/lib/moor_ffi.dart @@ -0,0 +1,7 @@ +import 'dart:io'; + +import 'package:moor/backends.dart'; +import 'package:moor/moor.dart'; +import 'package:moor_ffi/src/api/database.dart'; + +part 'src/vm_database.dart'; diff --git a/moor/lib/src/vm/api/database.dart b/moor_ffi/lib/src/api/database.dart similarity index 94% rename from moor/lib/src/vm/api/database.dart rename to moor_ffi/lib/src/api/database.dart index 97ad69de..96633e2a 100644 --- a/moor/lib/src/vm/api/database.dart +++ b/moor_ffi/lib/src/api/database.dart @@ -4,11 +4,11 @@ import 'dart:io'; import 'dart:typed_data'; import 'package:collection/collection.dart'; -import 'package:moor/src/vm/bindings/constants.dart'; -import 'package:moor/src/vm/bindings/types.dart' as types; -import 'package:moor/src/vm/bindings/bindings.dart'; -import 'package:moor/src/vm/ffi/blob.dart'; -import 'package:moor/src/vm/ffi/utils.dart'; +import 'package:moor_ffi/src/bindings/constants.dart'; +import 'package:moor_ffi/src/bindings/types.dart' as types; +import 'package:moor_ffi/src/bindings/bindings.dart'; +import 'package:moor_ffi/src/ffi/blob.dart'; +import 'package:moor_ffi/src/ffi/utils.dart'; part 'errors.dart'; part 'prepared_statement.dart'; diff --git a/moor/lib/src/vm/api/errors.dart b/moor_ffi/lib/src/api/errors.dart similarity index 100% rename from moor/lib/src/vm/api/errors.dart rename to moor_ffi/lib/src/api/errors.dart diff --git a/moor/lib/src/vm/api/prepared_statement.dart b/moor_ffi/lib/src/api/prepared_statement.dart similarity index 100% rename from moor/lib/src/vm/api/prepared_statement.dart rename to moor_ffi/lib/src/api/prepared_statement.dart diff --git a/moor/lib/src/vm/api/result.dart b/moor_ffi/lib/src/api/result.dart similarity index 100% rename from moor/lib/src/vm/api/result.dart rename to moor_ffi/lib/src/api/result.dart diff --git a/moor/lib/src/vm/bindings/bindings.dart b/moor_ffi/lib/src/bindings/bindings.dart similarity index 100% rename from moor/lib/src/vm/bindings/bindings.dart rename to moor_ffi/lib/src/bindings/bindings.dart diff --git a/moor/lib/src/vm/bindings/constants.dart b/moor_ffi/lib/src/bindings/constants.dart similarity index 100% rename from moor/lib/src/vm/bindings/constants.dart rename to moor_ffi/lib/src/bindings/constants.dart diff --git a/moor/lib/src/vm/bindings/signatures.dart b/moor_ffi/lib/src/bindings/signatures.dart similarity index 100% rename from moor/lib/src/vm/bindings/signatures.dart rename to moor_ffi/lib/src/bindings/signatures.dart diff --git a/moor/lib/src/vm/bindings/types.dart b/moor_ffi/lib/src/bindings/types.dart similarity index 100% rename from moor/lib/src/vm/bindings/types.dart rename to moor_ffi/lib/src/bindings/types.dart diff --git a/moor/lib/src/vm/ffi/blob.dart b/moor_ffi/lib/src/ffi/blob.dart similarity index 93% rename from moor/lib/src/vm/ffi/blob.dart rename to moor_ffi/lib/src/ffi/blob.dart index f1d9bce5..b562ec17 100644 --- a/moor/lib/src/vm/ffi/blob.dart +++ b/moor_ffi/lib/src/ffi/blob.dart @@ -3,7 +3,7 @@ import 'dart:ffi'; import 'dart:typed_data'; -import 'package:moor/src/vm/ffi/utils.dart'; +import 'package:moor_ffi/src/ffi/utils.dart'; /// Pointer to arbitrary blobs in C. class CBlob extends Struct { @@ -33,6 +33,7 @@ class CBlob extends Struct { final str = addressOf; if (isNullPointer(str)) return null; + // todo can we user Pointer.asExternalTypedData here? final blob = Uint8List(bytesToRead); for (var i = 0; i < bytesToRead; ++i) { blob[i] = str.elementAt(i).load().data; diff --git a/moor/lib/src/vm/ffi/open_platform_specific.dart b/moor_ffi/lib/src/ffi/open_platform_specific.dart similarity index 100% rename from moor/lib/src/vm/ffi/open_platform_specific.dart rename to moor_ffi/lib/src/ffi/open_platform_specific.dart diff --git a/moor/lib/src/vm/ffi/utils.dart b/moor_ffi/lib/src/ffi/utils.dart similarity index 100% rename from moor/lib/src/vm/ffi/utils.dart rename to moor_ffi/lib/src/ffi/utils.dart diff --git a/moor/lib/src/vm/vm_database.dart b/moor_ffi/lib/src/vm_database.dart similarity index 98% rename from moor/lib/src/vm/vm_database.dart rename to moor_ffi/lib/src/vm_database.dart index 23af5542..4566c3f5 100644 --- a/moor/lib/src/vm/vm_database.dart +++ b/moor_ffi/lib/src/vm_database.dart @@ -1,4 +1,4 @@ -part of 'package:moor/moor_vm.dart'; +part of 'package:moor_ffi/moor_ffi.dart'; /// A moor database that runs on the Dart VM. class VMDatabase extends DelegatedDatabase { diff --git a/moor_ffi/pubspec.yaml b/moor_ffi/pubspec.yaml new file mode 100644 index 00000000..ea18902c --- /dev/null +++ b/moor_ffi/pubspec.yaml @@ -0,0 +1,25 @@ +name: moor_ffi +description: "Experimental moor implementation that uses dart:ffi" +version: 2.0.0-dev +author: +homepage: + +environment: + sdk: ">=2.5.0-dev <2.6.0" + +dependencies: + moor: ^2.0.0 +# flutter: +# sdk: flutter + +dev_dependencies: + test: ^1.6.0 + +#flutter: +# plugin: +# androidPackage: com.example.moor_ffi +# pluginClass: MoorFfiPlugin + +dependency_overrides: + moor: + path: ../moor \ No newline at end of file From ec0e82eae46fd7d622ab0d3c42bd1080c5b845f7 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 21:07:27 +0200 Subject: [PATCH 091/117] Allow users to override library opening behavior --- extras/integration_tests/vm/pubspec.yaml | 6 +++++ extras/integration_tests/vm/test/vm_test.dart | 2 +- moor_ffi/README.md | 4 +-- moor_ffi/lib/moor_ffi.dart | 2 ++ moor_ffi/lib/src/bindings/bindings.dart | 5 ++-- .../lib/src/ffi/open_platform_specific.dart | 23 ---------------- moor_ffi/lib/src/load_library.dart | 26 +++++++++++++++++++ 7 files changed, 40 insertions(+), 28 deletions(-) delete mode 100644 moor_ffi/lib/src/ffi/open_platform_specific.dart create mode 100644 moor_ffi/lib/src/load_library.dart diff --git a/extras/integration_tests/vm/pubspec.yaml b/extras/integration_tests/vm/pubspec.yaml index fd4f4390..9f0ba3ae 100644 --- a/extras/integration_tests/vm/pubspec.yaml +++ b/extras/integration_tests/vm/pubspec.yaml @@ -8,8 +8,14 @@ environment: sdk: '>=2.4.0 <3.0.0' dependencies: + moor_ffi: + path: ../../../moor_ffi tests: path: ../tests dev_dependencies: test: ^1.5.0 + +dependency_overrides: + moor: + path: ../../../moor diff --git a/extras/integration_tests/vm/test/vm_test.dart b/extras/integration_tests/vm/test/vm_test.dart index a3fb763b..dbe1d991 100644 --- a/extras/integration_tests/vm/test/vm_test.dart +++ b/extras/integration_tests/vm/test/vm_test.dart @@ -1,6 +1,6 @@ import 'dart:io'; -import 'package:moor/moor_vm.dart'; +import 'package:moor_ffi/moor_ffi.dart'; import 'package:tests/tests.dart'; import 'package:path/path.dart' show join; diff --git a/moor_ffi/README.md b/moor_ffi/README.md index 80295ad6..6ac8655e 100644 --- a/moor_ffi/README.md +++ b/moor_ffi/README.md @@ -5,10 +5,10 @@ Moor backend that uses the new `dart:ffi` apis. ## Supported platforms At the moment, this plugin supports Android natively. However, it's also going to run on all platforms that expose `sqlite3` as a shared native library (macOS and virtually all Linux -distros). +distros, I'm not sure about Windows). Native iOS support is planned. ## Notes Using `flutter run` or `flutter build` when this library is imported is going to take very long for the first time. The reason is that we compile sqlite. Subsequent builds should take an acceptable -time execute. \ No newline at end of file +time execute. diff --git a/moor_ffi/lib/moor_ffi.dart b/moor_ffi/lib/moor_ffi.dart index c2d05425..632b3f81 100644 --- a/moor_ffi/lib/moor_ffi.dart +++ b/moor_ffi/lib/moor_ffi.dart @@ -1,3 +1,4 @@ +import 'dart:ffi'; import 'dart:io'; import 'package:moor/backends.dart'; @@ -5,3 +6,4 @@ import 'package:moor/moor.dart'; import 'package:moor_ffi/src/api/database.dart'; part 'src/vm_database.dart'; +part 'src/load_library.dart'; diff --git a/moor_ffi/lib/src/bindings/bindings.dart b/moor_ffi/lib/src/bindings/bindings.dart index 6df0a945..7008699d 100644 --- a/moor_ffi/lib/src/bindings/bindings.dart +++ b/moor_ffi/lib/src/bindings/bindings.dart @@ -4,8 +4,9 @@ import 'dart:ffi'; +import 'package:moor_ffi/moor_ffi.dart'; + import '../ffi/blob.dart'; -import '../ffi/open_platform_specific.dart'; import 'signatures.dart'; import 'types.dart'; @@ -93,7 +94,7 @@ class _SQLiteBindings { int Function(Pointer statement, int columnIndex) sqlite3_bind_null; _SQLiteBindings() { - sqlite = dlopenPlatformSpecific('sqlite3'); + sqlite = moorSqliteOpener(); sqlite3_bind_double = sqlite .lookup>( diff --git a/moor_ffi/lib/src/ffi/open_platform_specific.dart b/moor_ffi/lib/src/ffi/open_platform_specific.dart deleted file mode 100644 index 76befe80..00000000 --- a/moor_ffi/lib/src/ffi/open_platform_specific.dart +++ /dev/null @@ -1,23 +0,0 @@ -import 'dart:ffi'; -import 'dart:io'; - -String _platformPath(String name, {String path}) { - final resolvedPath = path ?? ''; - - if (Platform.isLinux || Platform.isAndroid) { - return '${resolvedPath}lib$name.so'; - } - if (Platform.isMacOS) { - return '${resolvedPath}lib$name.dylib'; - } - if (Platform.isWindows) { - return '$resolvedPath$name.dll'; - } - - throw UnsupportedError('Platform not implemented'); -} - -DynamicLibrary dlopenPlatformSpecific(String name, {String path}) { - final resolvedPath = _platformPath(name, path: path); - return DynamicLibrary.open(resolvedPath); -} diff --git a/moor_ffi/lib/src/load_library.dart b/moor_ffi/lib/src/load_library.dart new file mode 100644 index 00000000..44df83e5 --- /dev/null +++ b/moor_ffi/lib/src/load_library.dart @@ -0,0 +1,26 @@ +part of 'package:moor_ffi/moor_ffi.dart'; + +/// Signature responsible for loading the dynamic sqlite3 library that moor will +/// use. +typedef OpenLibrary = DynamicLibrary Function(); + +/// The [OpenLibrary] function that will be used for the first time the native +/// library is requested. This can be overridden, but won't have an effect after +/// the library has been opened once (which happens when a `VmDatabase` is +/// instantiated). +OpenLibrary moorSqliteOpener = _defaultOpen; + +DynamicLibrary _defaultOpen() { + if (Platform.isLinux || Platform.isAndroid) { + return DynamicLibrary.open('libsqlite3.so'); + } + if (Platform.isMacOS) { + return DynamicLibrary.open('libsqlite3.dylib'); + } + if (Platform.isWindows) { + return DynamicLibrary.open('sqlite3.dll'); + } + + throw UnsupportedError( + 'moor_ffi does not support ${Platform.operatingSystem} yet'); +} From 30e44ecbacc76c1d81d8d583d8ff6807d3f716fe Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 21:20:42 +0200 Subject: [PATCH 092/117] Write more notes on how to use moor_ffi --- extras/integration_tests/vm/test/vm_test.dart | 2 +- moor_ffi/README.md | 34 +++++++++++++++---- moor_ffi/lib/src/vm_database.dart | 12 +++---- moor_ffi/pubspec.yaml | 2 +- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/extras/integration_tests/vm/test/vm_test.dart b/extras/integration_tests/vm/test/vm_test.dart index dbe1d991..0f76b5b3 100644 --- a/extras/integration_tests/vm/test/vm_test.dart +++ b/extras/integration_tests/vm/test/vm_test.dart @@ -11,7 +11,7 @@ class VmExecutor extends TestExecutor { @override QueryExecutor createExecutor() { - return VMDatabase(file); + return VmDatabase(file); } @override diff --git a/moor_ffi/README.md b/moor_ffi/README.md index 6ac8655e..9f25f0c5 100644 --- a/moor_ffi/README.md +++ b/moor_ffi/README.md @@ -1,14 +1,36 @@ # moor_ffi -Moor backend that uses the new `dart:ffi` apis. +Moor backend that uses the new `dart:ffi` apis. Note that, while we have integration tests +on this package, it depends on the `dart:ffi` apis, which are in "preview" status at the moment. +Thus, this library is not suited for production use. + +If you want to use moor on Android or iOS, see the [getting started guide](https://moor.simonbinder.eu/docs/getting-started/) +which recommends to use the [moor_flutter](https://pub.dev/packages/moor_flutter) package. +At the moment, this library is targeted for advanced moor users who want to try out the `ffi` +backend. ## Supported platforms At the moment, this plugin supports Android natively. However, it's also going to run on all platforms that expose `sqlite3` as a shared native library (macOS and virtually all Linux -distros, I'm not sure about Windows). Native iOS support is planned. +distros, I'm not sure about Windows). Native iOS and macOS support is planned. + +## Migrating from moor_flutter +Add both `moor` and `moor_ffi` to your pubspec. + +```yaml +dependencies: + moor: ^2.0.0 + moor_ffi: ^0.0.1 +dev_dependencies: + moor: ^2.0.0 +``` + +In your main database file, replace the `package:moor_flutter/moor_flutter.dart` import with +`package:moor/moor.dart` and `package:moor_ffi/moor_ffi.dart`. +In all other project files that use moor apis (e.g. a `Value` class for companions), just import `package:moor/moor.dart`. + +Finally, replace usages of `FlutterQueryExecutor` with `VmDatabase`. ## Notes - -Using `flutter run` or `flutter build` when this library is imported is going to take very long for -the first time. The reason is that we compile sqlite. Subsequent builds should take an acceptable -time execute. +After importing this library, the first Flutter build is going to take a very long time. The reason is that we're +compiling sqlite to bundle it with your app. Subsequent builds should take an acceptable time to execute. diff --git a/moor_ffi/lib/src/vm_database.dart b/moor_ffi/lib/src/vm_database.dart index 4566c3f5..c73a8a5e 100644 --- a/moor_ffi/lib/src/vm_database.dart +++ b/moor_ffi/lib/src/vm_database.dart @@ -1,19 +1,19 @@ part of 'package:moor_ffi/moor_ffi.dart'; /// A moor database that runs on the Dart VM. -class VMDatabase extends DelegatedDatabase { - VMDatabase._(DatabaseDelegate delegate, bool logStatements) +class VmDatabase extends DelegatedDatabase { + VmDatabase._(DatabaseDelegate delegate, bool logStatements) : super(delegate, isSequential: true, logStatements: logStatements); /// Creates a database that will store its result in the [file], creating it /// if it doesn't exist. - factory VMDatabase(File file, {bool logStatements = false}) { - return VMDatabase._(_VmDelegate(file), logStatements); + factory VmDatabase(File file, {bool logStatements = false}) { + return VmDatabase._(_VmDelegate(file), logStatements); } /// Creates a database won't persist its changes on disk. - factory VMDatabase.memory({bool logStatements = false}) { - return VMDatabase._(_VmDelegate(null), logStatements); + factory VmDatabase.memory({bool logStatements = false}) { + return VmDatabase._(_VmDelegate(null), logStatements); } } diff --git a/moor_ffi/pubspec.yaml b/moor_ffi/pubspec.yaml index ea18902c..23c06087 100644 --- a/moor_ffi/pubspec.yaml +++ b/moor_ffi/pubspec.yaml @@ -1,6 +1,6 @@ name: moor_ffi description: "Experimental moor implementation that uses dart:ffi" -version: 2.0.0-dev +version: 0.0.1 author: homepage: From ae8ed3aedf7938d5298ebea52ea2464b5c770ecd Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 21:30:38 +0200 Subject: [PATCH 093/117] Also run the ffi integration tests on android --- .../integration_tests/flutter_db/lib/main.dart | 17 +++++++++++++++++ .../integration_tests/flutter_db/pubspec.lock | 13 ++++++++++--- .../integration_tests/flutter_db/pubspec.yaml | 2 ++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/extras/integration_tests/flutter_db/lib/main.dart b/extras/integration_tests/flutter_db/lib/main.dart index ab7a78a1..9716ce4e 100644 --- a/extras/integration_tests/flutter_db/lib/main.dart +++ b/extras/integration_tests/flutter_db/lib/main.dart @@ -1,6 +1,7 @@ import 'dart:io'; import 'package:flutter/services.dart'; +import 'package:moor_ffi/moor_ffi.dart'; import 'package:tests/tests.dart'; import 'package:test/test.dart'; import 'package:moor_flutter/moor_flutter.dart'; @@ -27,8 +28,24 @@ class SqfliteExecutor extends TestExecutor { } } +class FfiExecutor extends TestExecutor { + @override + QueryExecutor createExecutor() { + return VmDatabase(File('app_ffi.db')); + } + + @override + Future deleteData() async { + final file = File('app_ffi.db'); + if (await file.exists()) { + await file.delete(); + } + } +} + void main() { runAllTests(SqfliteExecutor()); + runAllTests(FfiExecutor()); // Additional integration test for flutter: Test loading a database from asset test('can load a database from asset', () async { diff --git a/extras/integration_tests/flutter_db/pubspec.lock b/extras/integration_tests/flutter_db/pubspec.lock index 2b92d0f6..04a3d533 100644 --- a/extras/integration_tests/flutter_db/pubspec.lock +++ b/extras/integration_tests/flutter_db/pubspec.lock @@ -159,7 +159,7 @@ packages: name: meta url: "https://pub.dartlang.org" source: hosted - version: "1.1.6" + version: "1.1.7" mime: dependency: transitive description: @@ -173,7 +173,14 @@ packages: path: "../../../moor" relative: true source: path - version: "1.7.1" + version: "1.7.2" + moor_ffi: + dependency: "direct main" + description: + path: "../../../moor_ffi" + relative: true + source: path + version: "0.0.1" moor_flutter: dependency: "direct main" description: @@ -404,5 +411,5 @@ packages: source: hosted version: "2.1.16" sdks: - dart: ">=2.4.0 <3.0.0" + dart: ">=2.5.0-dev <2.6.0" flutter: ">=1.2.1 <2.0.0" diff --git a/extras/integration_tests/flutter_db/pubspec.yaml b/extras/integration_tests/flutter_db/pubspec.yaml index d0bb15d6..7b164fb1 100644 --- a/extras/integration_tests/flutter_db/pubspec.yaml +++ b/extras/integration_tests/flutter_db/pubspec.yaml @@ -11,6 +11,8 @@ dependencies: moor_flutter: tests: path: ../tests + moor_ffi: + path: ../../../moor_ffi dev_dependencies: test: From 082de160bc134097eb0173f8042e01c23dd20dc9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 21:54:16 +0200 Subject: [PATCH 094/117] Start to write changelog for the next version --- moor/CHANGELOG.md | 54 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 2accea65..1c2db95e 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,13 +1,55 @@ ## 2.0.0 -This is the first major update after the initial release and moor and we have a lot to cover. -... Finally, we also removed a variety of deprecated features. See the breaking changes +This is the first major update after the initial release and moor and we have a lot to cover: +The `.moor` files can now have their own imports and queries, you can embed Dart in sql queries +using the new templates feature and we have a prototype of a pure-Dart SQL IDE ready. +Finally, we also removed a variety of deprecated features. See the breaking changes section to learn what components are affected and what alternatives are available. -TODO: Properly describe these additions when they're finalized: +### New features -- Queries and imports in `.moor` files -- Analyzer plugin for Dart Code -- `ffi` libraries +#### Updates to the sql parser +`.moor` files were introduced in moor 1.7 as an experimental way to declare tables by using +`CREATE TABLE` statements. In this version, they become stable and support their own import +and query system. This allows you to write queries in their own file: + +```sql +CREATE TABLE users ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL +); + +findByName: SELECT * FROM users WHERE name LIKE :query; +``` +When this file is included from a `@UseMoor` annotation, moor will generate methods to run the +query. Of course, you can also write Dart queries for tables declared in sql: +```dart +Stream loadUserById(int id) { + return (select(users)..where((u) => u.id.equals(2))).watchSingle(); +} +``` + +Moor files can also import other moor files by using an `import 'other.moor';'` statement at the +top. Then, all tables defined in `other.moor` will also be available to the current file. + +Moor takes Dart and SQL interop even further with the new "Dart in SQL templates". You can define +a query like this: +```sql +findDynamic: SELECT * FROM users WHERE $condition; +``` + +And moor will generate a method `findDynamic(Expression condition)` for you. This +allows you to bind the template with a predicate as complex as you'd like. At the moment, Dart +templates are supported for expressions, `OrderBy`, `OrderingTerm` and `Limit`. + +`INSERT` statements can now be used as a compiled statement - both in moor files and +in a `@UseMoor` or `@UseDao` annotation. A new builtin linter will even warn you when you forget +to provide a value for a non-nullable column - right at compile time! + +And finally, we now generate better query code when queries only return a single column. Instead of +generating a whole new class for that, we simply return the value directly. + +#### Experimental ffi support +TODO: Describe ffi port ### Minor changes - a `Constant` can now be written to SQL, it used to throw before. This is useful From c95a3badb252d03f77a9172fd63de3a451ed50db Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 14 Sep 2019 22:44:20 +0200 Subject: [PATCH 095/117] Infer types for dart placeholder expressions Also, infer where clauses to have a bool type --- moor_generator/lib/src/model/sql_query.dart | 7 ++++--- sqlparser/lib/src/analysis/types/resolver.dart | 6 ++++-- sqlparser/lib/src/ast/statements/delete.dart | 5 ++++- sqlparser/lib/src/ast/statements/select.dart | 5 ++++- sqlparser/lib/src/ast/statements/statement.dart | 6 ++++++ sqlparser/lib/src/ast/statements/update.dart | 5 ++++- sqlparser/test/analysis/type_resolver_test.dart | 11 +++++++++++ 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index b5cd4a68..a5ede39a 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -137,10 +137,11 @@ class InferredResultSet { /// Whether a new class needs to be written to store the result of this query. /// We don't need to do that for queries which return an existing table model /// or if they only return exactly one column. - bool get needsOwnClass => matchingTable == null && !singleColumn; + bool get needsOwnClass => matchingTable == null && columns.length > 1; - /// Whether this query returns a single column. - bool get singleColumn => columns.length == 1; + /// Whether this query returns a single column that should be returned + /// directly. + bool get singleColumn => matchingTable == null && columns.length == 1; void forceDartNames(Map names) { _dartNames diff --git a/sqlparser/lib/src/analysis/types/resolver.dart b/sqlparser/lib/src/analysis/types/resolver.dart index 4a5cc35e..9b0f0525 100644 --- a/sqlparser/lib/src/analysis/types/resolver.dart +++ b/sqlparser/lib/src/analysis/types/resolver.dart @@ -37,8 +37,8 @@ class TypeResolver { ResolveResult resolveOrInfer(Typeable t) { if (t is Column) { return resolveColumn(t); - } else if (t is Variable) { - return inferType(t); + } else if (t is Variable || t is DartExpressionPlaceholder) { + return inferType(t as Expression); } else if (t is Expression) { return resolveExpression(t); } @@ -297,6 +297,8 @@ class TypeResolver { // appears as part of a bounded window definition: // RANGE BETWEEN PRECEDING AND FOLLOWING return const ResolveResult(ResolvedType(type: BasicType.int)); + } else if (parent is HasWhereClause && e == parent.where) { + return const ResolveResult(ResolvedType.bool()); } return const ResolveResult.unknown(); diff --git a/sqlparser/lib/src/ast/statements/delete.dart b/sqlparser/lib/src/ast/statements/delete.dart index a6326e41..450d48f8 100644 --- a/sqlparser/lib/src/ast/statements/delete.dart +++ b/sqlparser/lib/src/ast/statements/delete.dart @@ -1,7 +1,10 @@ part of '../ast.dart'; -class DeleteStatement extends Statement with CrudStatement { +class DeleteStatement extends Statement + with CrudStatement + implements HasWhereClause { final TableReference from; + @override final Expression where; DeleteStatement({@required this.from, this.where}); diff --git a/sqlparser/lib/src/ast/statements/select.dart b/sqlparser/lib/src/ast/statements/select.dart index a3ed4d7d..4c4e52c5 100644 --- a/sqlparser/lib/src/ast/statements/select.dart +++ b/sqlparser/lib/src/ast/statements/select.dart @@ -1,10 +1,13 @@ part of '../ast.dart'; -class SelectStatement extends Statement with CrudStatement, ResultSet { +class SelectStatement extends Statement + with CrudStatement, ResultSet + implements HasWhereClause { final bool distinct; final List columns; final List from; + @override final Expression where; final GroupBy groupBy; final List windowDeclarations; diff --git a/sqlparser/lib/src/ast/statements/statement.dart b/sqlparser/lib/src/ast/statements/statement.dart index 603352c0..f46d0750 100644 --- a/sqlparser/lib/src/ast/statements/statement.dart +++ b/sqlparser/lib/src/ast/statements/statement.dart @@ -7,5 +7,11 @@ abstract class Statement extends AstNode { /// Marker mixin for statements that read from an existing table structure. mixin CrudStatement on Statement {} +/// Interface for statements that have a primary where clause (select, update, +/// delete). +abstract class HasWhereClause extends Statement { + Expression get where; +} + /// Marker mixin for statements that change the table structure. mixin SchemaStatement on Statement {} diff --git a/sqlparser/lib/src/ast/statements/update.dart b/sqlparser/lib/src/ast/statements/update.dart index a4bc0f8c..f7c8877f 100644 --- a/sqlparser/lib/src/ast/statements/update.dart +++ b/sqlparser/lib/src/ast/statements/update.dart @@ -16,10 +16,13 @@ const Map _tokensToMode = { TokenType.ignore: FailureMode.ignore, }; -class UpdateStatement extends Statement with CrudStatement { +class UpdateStatement extends Statement + with CrudStatement + implements HasWhereClause { final FailureMode or; final TableReference table; final List set; + @override final Expression where; UpdateStatement( diff --git a/sqlparser/test/analysis/type_resolver_test.dart b/sqlparser/test/analysis/type_resolver_test.dart index 0e1a2f1d..a323a944 100644 --- a/sqlparser/test/analysis/type_resolver_test.dart +++ b/sqlparser/test/analysis/type_resolver_test.dart @@ -73,4 +73,15 @@ void main() { expect(ctx.typeOf(secondVar), equals(const ResolveResult(ResolvedType(type: BasicType.text)))); }); + + test('can infer types for dart placeholder', () { + final ctx = (SqlEngine(useMoorExtensions: true)..registerTable(demoTable)) + .analyze(r'SELECT * FROM demo WHERE $expr'); + + final dartExpr = + ctx.root.allDescendants.whereType().single; + + expect(ctx.typeOf(dartExpr as Expression), + const ResolveResult(ResolvedType.bool())); + }); } From 5b6bd1db37a243d3f23cc07e1b518e634f8034c9 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 15 Sep 2019 10:51:35 +0200 Subject: [PATCH 096/117] Fix some queries with Dart templates not compiling --- moor/lib/moor.dart | 1 + .../lib/src/runtime/expressions/user_api.dart | 1 + moor/test/data/tables/custom_tables.g.dart | 8 ++++++++ moor/test/data/tables/tables.moor | 3 ++- .../moor_files_integration_test.dart | 16 +++++++++++++++ .../lib/src/writer/queries/query_writer.dart | 20 +++++++++++-------- 6 files changed, 40 insertions(+), 9 deletions(-) diff --git a/moor/lib/moor.dart b/moor/lib/moor.dart index ef704350..6621ad3d 100644 --- a/moor/lib/moor.dart +++ b/moor/lib/moor.dart @@ -13,6 +13,7 @@ export 'package:moor/src/dsl/database.dart'; export 'package:moor/src/runtime/components/join.dart' show innerJoin, leftOuterJoin, crossJoin; +export 'package:moor/src/runtime/components/limit.dart'; export 'package:moor/src/runtime/components/order_by.dart'; export 'package:moor/src/runtime/executor/executor.dart'; export 'package:moor/src/types/type_system.dart'; diff --git a/moor/lib/src/runtime/expressions/user_api.dart b/moor/lib/src/runtime/expressions/user_api.dart index ea0a6b93..bcdff58b 100644 --- a/moor/lib/src/runtime/expressions/user_api.dart +++ b/moor/lib/src/runtime/expressions/user_api.dart @@ -1,6 +1,7 @@ export 'bools.dart' show and, or, not; export 'custom.dart'; export 'datetimes.dart'; +export 'expression.dart' show Expression; export 'in.dart'; export 'null_check.dart'; export 'text.dart' show Collate; diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index dea654b6..5dca3282 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -842,6 +842,14 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { }).map(_rowToConfigData); } + Selectable readDynamic(Expression predicate) { + final generatedpredicate = $write(predicate); + return customSelectQuery( + 'SELECT * FROM config WHERE ${generatedpredicate.sql}', + variables: [...generatedpredicate.introducedVariables], + readsFrom: {config}).map(_rowToConfigData); + } + Future writeConfig(String key, String value) { return customInsert( 'REPLACE INTO config VALUES (:key, :value)', diff --git a/moor/test/data/tables/tables.moor b/moor/test/data/tables/tables.moor index df9164be..e4676550 100644 --- a/moor/test/data/tables/tables.moor +++ b/moor/test/data/tables/tables.moor @@ -26,4 +26,5 @@ CREATE TABLE mytable ( ); readConfig: SELECT * FROM config WHERE config_key = ?; -readMultiple: SELECT * FROM config WHERE config_key IN ? ORDER BY $clause; \ No newline at end of file +readMultiple: SELECT * FROM config WHERE config_key IN ? ORDER BY $clause; +readDynamic: SELECT * FROM config WHERE $predicate; \ No newline at end of file diff --git a/moor/test/parsed_sql/moor_files_integration_test.dart b/moor/test/parsed_sql/moor_files_integration_test.dart index a10a460b..730b1963 100644 --- a/moor/test/parsed_sql/moor_files_integration_test.dart +++ b/moor/test/parsed_sql/moor_files_integration_test.dart @@ -67,4 +67,20 @@ void main() { ['a', 'b'], )); }); + + test('runs query with variables from template', () async { + final mock = MockExecutor(); + final db = CustomTablesDb(mock); + + final mockResponse = {'config_key': 'key', 'config_value': 'value'}; + when(mock.runSelect(any, any)) + .thenAnswer((_) => Future.value([mockResponse])); + + final parsed = + await db.readDynamic(db.config.configKey.equals('key')).getSingle(); + + verify( + mock.runSelect('SELECT * FROM config WHERE config_key = ?', ['key'])); + expect(parsed, ConfigData(configKey: 'key', configValue: 'value')); + }); } diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index f2fec6c5..54ddd887 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -244,9 +244,14 @@ class QueryWriter { void _writeExpandedDeclarations() { var indexCounterWasDeclared = false; + final needsIndexCounter = query.variables.any((v) => v.isArray); var highestIndexBeforeArray = 0; - void _writeIndexCounter() { + void _writeIndexCounterIfNeeded() { + if (indexCounterWasDeclared || !needsIndexCounter) { + return; // already written or not necessary at all + } + // we only need the index counter when the query contains an expanded // element. // add +1 because that's going to be the first index of this element. @@ -256,7 +261,9 @@ class QueryWriter { } void _increaseIndexCounter(String by) { - _buffer..write('$highestAssignedIndexVar += ')..write(by)..write(';\n'); + if (needsIndexCounter) { + _buffer..write('$highestAssignedIndexVar += ')..write(by)..write(';\n'); + } } // query.elements are guaranteed to be sorted in the order in which they're @@ -265,9 +272,7 @@ class QueryWriter { for (var element in query.elements) { if (element is FoundVariable) { if (element.isArray) { - if (!indexCounterWasDeclared) { - _writeIndexCounter(); - } + _writeIndexCounterIfNeeded(); // final expandedvar1 = $expandVar(, ); _buffer @@ -288,9 +293,8 @@ class QueryWriter { highestIndexBeforeArray = max(highestIndexBeforeArray, element.index); } } else if (element is FoundDartPlaceholder) { - if (!indexCounterWasDeclared) { - indexCounterWasDeclared = true; - } + _writeIndexCounterIfNeeded(); + _buffer ..write('final ') ..write(_placeholderContextName(element)) From 3abfbd59639cbace0040dad0a9e469f1772a9b9b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 15 Sep 2019 11:19:32 +0200 Subject: [PATCH 097/117] Don't parse tuples as expressions --- analysis_options.yaml | 1 - .../lib/src/analysis/types/resolver.dart | 4 +- sqlparser/lib/src/ast/ast.dart | 6 +- sqlparser/lib/src/ast/common/tuple.dart | 22 +++++++ sqlparser/lib/src/ast/expressions/simple.dart | 10 ++- sqlparser/lib/src/ast/expressions/tuple.dart | 19 ------ sqlparser/lib/src/ast/statements/insert.dart | 2 +- sqlparser/lib/src/reader/parser/crud.dart | 2 +- .../lib/src/reader/parser/expressions.dart | 65 ++++++++----------- sqlparser/lib/src/reader/parser/parser.dart | 2 +- sqlparser/test/parser/expression_test.dart | 6 -- sqlparser/test/parser/insert_test.dart | 2 +- .../test/parser/select/generic_test.dart | 2 +- 13 files changed, 63 insertions(+), 80 deletions(-) create mode 100644 sqlparser/lib/src/ast/common/tuple.dart delete mode 100644 sqlparser/lib/src/ast/expressions/tuple.dart diff --git a/analysis_options.yaml b/analysis_options.yaml index 1793a810..f927bc1c 100644 --- a/analysis_options.yaml +++ b/analysis_options.yaml @@ -7,7 +7,6 @@ analyzer: unused_local_variable: error dead_code: error override_on_non_overriding_method: error - deprecated_member_use_from_same_package: ignore exclude: - "**/*.g.dart" # Will be analyzed anyway, nobody knows why ĀÆ\_(惄)_/ĀÆ. We're only analyzing lib/ and test/ as a workaround diff --git a/sqlparser/lib/src/analysis/types/resolver.dart b/sqlparser/lib/src/analysis/types/resolver.dart index 9b0f0525..0794bba1 100644 --- a/sqlparser/lib/src/analysis/types/resolver.dart +++ b/sqlparser/lib/src/analysis/types/resolver.dart @@ -285,7 +285,7 @@ class TypeResolver { // If this appears in a tuple, e.g. test IN (?). The "(?)" will be an // array. Of course, the individual entry is not, so reset that state. - if (parent is TupleExpression) { + if (parent is Tuple) { return inferredType.mapResult((r) => r.toArray(false)); } return inferredType; @@ -330,7 +330,7 @@ class TypeResolver { return resolveExpression(otherNode as Expression); } } else if (parent is Parentheses || - parent is TupleExpression || + parent is Tuple || parent is UnaryExpression) { return const ResolveResult.needsContext(); } else if (parent is Invocation) { diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index edeb6953..a78af758 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -9,6 +9,7 @@ part 'clauses/ordering.dart'; part 'common/queryables.dart'; part 'common/renamable.dart'; +part 'common/tuple.dart'; part 'expressions/aggregate.dart'; part 'expressions/case.dart'; @@ -18,7 +19,6 @@ part 'expressions/literals.dart'; part 'expressions/reference.dart'; part 'expressions/simple.dart'; part 'expressions/subquery.dart'; -part 'expressions/tuple.dart'; part 'expressions/variables.dart'; part 'moor/declared_statement.dart'; @@ -180,7 +180,7 @@ abstract class AstVisitor { T visitExists(ExistsExpression e); T visitCaseExpression(CaseExpression e); T visitWhen(WhenComponent e); - T visitTuple(TupleExpression e); + T visitTuple(Tuple e); T visitInExpression(InExpression e); T visitAggregateExpression(AggregateExpression e); @@ -223,7 +223,7 @@ class RecursiveVisitor extends AstVisitor { T visitWhen(WhenComponent e) => visitChildren(e); @override - T visitTuple(TupleExpression e) => visitChildren(e); + T visitTuple(Tuple e) => visitChildren(e); @override T visitInExpression(InExpression e) => visitChildren(e); diff --git a/sqlparser/lib/src/ast/common/tuple.dart b/sqlparser/lib/src/ast/common/tuple.dart new file mode 100644 index 00000000..5732af9c --- /dev/null +++ b/sqlparser/lib/src/ast/common/tuple.dart @@ -0,0 +1,22 @@ +part of '../ast.dart'; + +/// A tuple of values, denotes in brackets. `(, ..., )`. +/// +/// Notice that this class extends [Expression] because the type inference +/// algorithm works best when tuples are treated as expressions. Syntactically, +/// tuples aren't expressions. +class Tuple extends Expression { + /// The expressions appearing in this tuple. + final List expressions; + + Tuple({@required this.expressions}); + + @override + T accept(AstVisitor visitor) => visitor.visitTuple(this); + + @override + Iterable get childNodes => expressions; + + @override + bool contentEquals(Tuple other) => true; +} diff --git a/sqlparser/lib/src/ast/expressions/simple.dart b/sqlparser/lib/src/ast/expressions/simple.dart index bfa038bb..48bb0c81 100644 --- a/sqlparser/lib/src/ast/expressions/simple.dart +++ b/sqlparser/lib/src/ast/expressions/simple.dart @@ -118,13 +118,15 @@ class BetweenExpression extends Expression { bool contentEquals(BetweenExpression other) => other.not == not; } -/// `$left$ IN $inside` +/// `$left$ IN $inside`. class InExpression extends Expression { final bool not; final Expression left; final Expression inside; - InExpression({this.not = false, @required this.left, @required this.inside}); + InExpression({this.not = false, @required this.left, @required this.inside}) { + assert(inside is Tuple || inside is Variable); + } @override T accept(AstVisitor visitor) => visitor.visitInExpression(this); @@ -158,8 +160,4 @@ class Parentheses extends Expression { @override bool contentEquals(Parentheses other) => true; - - TupleExpression get asTuple { - return TupleExpression(expressions: [expression]); - } } diff --git a/sqlparser/lib/src/ast/expressions/tuple.dart b/sqlparser/lib/src/ast/expressions/tuple.dart deleted file mode 100644 index ebdc99b9..00000000 --- a/sqlparser/lib/src/ast/expressions/tuple.dart +++ /dev/null @@ -1,19 +0,0 @@ -part of '../ast.dart'; - -/// A tuple of values. For instance, in `SELECT * FROM t WHERE id IN (1,2,3)`, -/// the `(1,2,3)` is a tuple. -class TupleExpression extends Expression { - /// The expressions appearing in this tuple. - final List expressions; - - TupleExpression({@required this.expressions}); - - @override - T accept(AstVisitor visitor) => visitor.visitTuple(this); - - @override - Iterable get childNodes => expressions; - - @override - bool contentEquals(TupleExpression other) => true; -} diff --git a/sqlparser/lib/src/ast/statements/insert.dart b/sqlparser/lib/src/ast/statements/insert.dart index f4385526..e194bff8 100644 --- a/sqlparser/lib/src/ast/statements/insert.dart +++ b/sqlparser/lib/src/ast/statements/insert.dart @@ -72,7 +72,7 @@ abstract class InsertSource { /// Uses a list of values for an insert statement (`VALUES (a, b, c)`). class ValuesSource extends InsertSource { - final List values; + final List values; ValuesSource(this.values); diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index bad78ebc..9edca466 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -458,7 +458,7 @@ mixin CrudParser on ParserBase { InsertSource _insertSource() { if (_matchOne(TokenType.$values)) { - final values = []; + final values = []; do { values.add(_consumeTuple()); } while (_matchOne(TokenType.comma)); diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index e256ad7f..ff6b4fc9 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -67,13 +67,7 @@ mixin ExpressionParser on ParserBase { final not = _matchOne(TokenType.not); _matchOne(TokenType.$in); - var inside = _equals(); - if (inside is Parentheses) { - // if we have something like x IN (3), then (3) is a tuple and not a - // parenthesis. We can only know this from the context unfortunately - inside = (inside as Parentheses).asTuple; - } - + final inside = _variableOrNull() ?? _consumeTuple(); return InExpression(left: left, inside: inside, not: not); } @@ -248,42 +242,24 @@ mixin ExpressionParser on ParserBase { final literal = _literalOrNull(); if (literal != null) return literal; + final variable = _variableOrNull(); + if (variable != null) return variable; + final token = _advance(); final type = token.type; switch (type) { case TokenType.leftParen: - // Opening brackets could be three things: An inner select statement - // (SELECT ...), a parenthesised expression, or a tuple of expressions - // (a, b, c). + // An opening bracket in the context of an expression could either be + // an inner select statement or a parenthesised expression. final left = token; if (_peek.type == TokenType.select) { final stmt = select(); _consume(TokenType.rightParen, 'Expected a closing bracket'); return SubQuery(select: stmt); } else { - // alright, it's either a tuple or just parenthesis. A tuple can be - // empty, so if the next statement is the closing bracket we're done - if (_matchOne(TokenType.rightParen)) { - return TupleExpression(expressions: [])..setSpan(left, _previous); - } - final expr = expression(); - - // Are we witnessing a tuple? - if (_check(TokenType.comma)) { - // we are, add expressions as long as we see commas - final exprs = [expr]; - while (_matchOne(TokenType.comma)) { - exprs.add(expression()); - } - - _consume(TokenType.rightParen, 'Expected a closing bracket'); - return TupleExpression(expressions: exprs); - } else { - // we aren't, so that'll just be parentheses. - _consume(TokenType.rightParen, 'Expected a closing bracket'); - return Parentheses(left, expr, token); - } + _consume(TokenType.rightParen, 'Expected a closing bracket'); + return Parentheses(left, expr, token); } break; case TokenType.identifier: @@ -336,6 +312,17 @@ mixin ExpressionParser on ParserBase { _error('Could not parse this expression'); } + Variable _variableOrNull() { + if (_matchOne(TokenType.questionMarkVariable)) { + return NumberedVariable(_previous as QuestionMarkVariableToken) + ..setSpan(_previous, _previous); + } else if (_matchOne(TokenType.colonVariable)) { + return ColonNamedVariable(_previous as ColonVariableToken) + ..setSpan(_previous, _previous); + } + return null; + } + FunctionParameters _functionParameters() { if (_matchOne(TokenType.star)) { return const StarFunctionParameter(); @@ -390,18 +377,20 @@ mixin ExpressionParser on ParserBase { } @override - TupleExpression _consumeTuple() { + Tuple _consumeTuple() { final firstToken = _consume(TokenType.leftParen, 'Expected opening parenthesis for tuple'); final expressions = []; - do { - expressions.add(expression()); - } while (_matchOne(TokenType.comma)); + // tuples can be empty `()`, so only start parsing values when it's not + if (_peek.type != TokenType.rightParen) { + do { + expressions.add(expression()); + } while (_matchOne(TokenType.comma)); + } _consume(TokenType.rightParen, 'Expected right parenthesis to close tuple'); - return TupleExpression(expressions: expressions) - ..setSpan(firstToken, _previous); + return Tuple(expressions: expressions)..setSpan(firstToken, _previous); } } diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 007392a3..237a3e89 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -142,7 +142,7 @@ abstract class ParserBase { // Common operations that we are referenced very often Expression expression(); - TupleExpression _consumeTuple(); + Tuple _consumeTuple(); /// Parses a [SelectStatement], or returns null if there is no select token /// after the current position. diff --git a/sqlparser/test/parser/expression_test.dart b/sqlparser/test/parser/expression_test.dart index f0508dc2..3e62aeb8 100644 --- a/sqlparser/test/parser/expression_test.dart +++ b/sqlparser/test/parser/expression_test.dart @@ -83,12 +83,6 @@ final Map _testCases = { ), ), ), - '(1, 2, 3)': TupleExpression( - expressions: [ - for (var i = 1; i <= 3; i++) - NumericLiteral(i, token(TokenType.numberLiteral)), - ], - ), "'hello' || 'world' COLLATE NOCASE": BinaryExpression( StringLiteral.from(token(TokenType.stringLiteral), 'hello'), token(TokenType.doublePipe), diff --git a/sqlparser/test/parser/insert_test.dart b/sqlparser/test/parser/insert_test.dart index 75580cb5..2e5bc6c5 100644 --- a/sqlparser/test/parser/insert_test.dart +++ b/sqlparser/test/parser/insert_test.dart @@ -16,7 +16,7 @@ void main() { Reference(columnName: 'c'), ], source: ValuesSource([ - TupleExpression(expressions: [ + Tuple(expressions: [ Reference(columnName: 'd'), Reference(columnName: 'e'), Reference(columnName: 'f'), diff --git a/sqlparser/test/parser/select/generic_test.dart b/sqlparser/test/parser/select/generic_test.dart index 75917947..bd4c9b12 100644 --- a/sqlparser/test/parser/select/generic_test.dart +++ b/sqlparser/test/parser/select/generic_test.dart @@ -46,7 +46,7 @@ final Map testCases = { from: [TableReference('tbl', null)], where: InExpression( left: Reference(columnName: 'id'), - inside: TupleExpression(expressions: []), + inside: Tuple(expressions: []), ), ), }; From 25ceda35050976355ed8aec8f8358bdab2fd6f89 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 15 Sep 2019 11:59:47 +0200 Subject: [PATCH 098/117] Report lints about invalid Dart Templates in SQL --- moor/test/database_test.dart | 5 +-- .../analyzer/sql_queries/lints/linter.dart | 35 +++++++++++++++++++ .../analyzer/sql_queries/query_handler.dart | 4 +-- .../lib/src/writer/queries/query_writer.dart | 3 +- .../analyzer/sql_queries/linter_test.dart | 25 +++++++++++++ .../lib/src/analysis/types/resolver.dart | 2 +- 6 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 moor_generator/test/analyzer/sql_queries/linter_test.dart diff --git a/moor/test/database_test.dart b/moor/test/database_test.dart index bdb256fc..bf94e08f 100644 --- a/moor/test/database_test.dart +++ b/moor/test/database_test.dart @@ -17,8 +17,9 @@ class _FakeDb extends GeneratedDatabase { }, beforeOpen: (details) async { // this fake select query is verified via mocks - await customSelect( - 'opened: ${details.versionBefore} to ${details.versionNow}'); + await customSelectQuery( + 'opened: ${details.versionBefore} to ${details.versionNow}') + .get(); }, ); } diff --git a/moor_generator/lib/src/analyzer/sql_queries/lints/linter.dart b/moor_generator/lib/src/analyzer/sql_queries/lints/linter.dart index 69f2dccf..b9648f4e 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/lints/linter.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/lints/linter.dart @@ -2,6 +2,8 @@ import 'package:sqlparser/sqlparser.dart'; import '../query_handler.dart'; +/// Provides additional hints that aren't implemented in the sqlparser because +/// they're specific to moor. class Linter { final QueryHandler handler; final List lints = []; @@ -18,6 +20,39 @@ class _LintingVisitor extends RecursiveVisitor { _LintingVisitor(this.linter); + @override + void visitResultColumn(ResultColumn e) { + super.visitResultColumn(e); + + if (e is ExpressionResultColumn) { + // The generated code will be invalid if knowing the expression is needed + // to know the column name (e.g. it's a Dart template without an AS), or + // if the type is unknown. + final expression = e.expression; + final resolveResult = linter.handler.context.typeOf(expression); + + if (resolveResult.type == null) { + linter.lints.add(AnalysisError( + type: AnalysisErrorType.other, + message: 'Expression has an unknown type, the generated code can be' + ' inaccurate.', + relevantNode: expression, + )); + } + + final dependsOnPlaceholder = e.as == null && + expression.allDescendants.whereType().isNotEmpty; + if (dependsOnPlaceholder) { + linter.lints.add(AnalysisError( + type: AnalysisErrorType.other, + message: 'The name of this column depends on a Dart template, which ' + 'breaks generated code. Try adding an `AS` alias to fix this.', + relevantNode: e, + )); + } + } + } + @override void visitInsertStatement(InsertStatement e) { final targeted = e.resolvedTargetColumns; diff --git a/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart index 7ac3253a..e581c3df 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/query_handler.dart @@ -82,11 +82,11 @@ class QueryHandler { final type = context.typeOf(column).type; final moorType = mapper.resolvedToMoor(type); UsedTypeConverter converter; - if (type.hint is TypeConverterHint) { + if (type?.hint is TypeConverterHint) { converter = (type.hint as TypeConverterHint).converter; } - columns.add(ResultColumn(column.name, moorType, type.nullable, + columns.add(ResultColumn(column.name, moorType, type?.nullable ?? true, converter: converter)); final table = _tableOfColumn(column); diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 54ddd887..b93ad713 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -113,7 +113,8 @@ class QueryWriter { String _readingCode(ResultColumn column) { final readMethod = readFromMethods[column.type]; - var code = "row.$readMethod('${column.name}')"; + final dartLiteral = asDartLiteral(column.name); + var code = 'row.$readMethod($dartLiteral)'; if (column.converter != null) { final converter = column.converter; diff --git a/moor_generator/test/analyzer/sql_queries/linter_test.dart b/moor_generator/test/analyzer/sql_queries/linter_test.dart new file mode 100644 index 00000000..67225cca --- /dev/null +++ b/moor_generator/test/analyzer/sql_queries/linter_test.dart @@ -0,0 +1,25 @@ +import 'package:moor_generator/src/analyzer/sql_queries/query_handler.dart'; +import 'package:moor_generator/src/analyzer/sql_queries/type_mapping.dart'; +import 'package:sqlparser/sqlparser.dart'; +import 'package:test/test.dart'; + +void main() { + final engine = SqlEngine(useMoorExtensions: true); + final mapper = TypeMapper(); + + test('warns when a result column is unresolved', () { + final result = engine.analyze('SELECT ?;'); + final moorQuery = QueryHandler('query', result, mapper).handle(); + + expect(moorQuery.lints, + anyElement((AnalysisError q) => q.message.contains('unknown type'))); + }); + + test('warns when the result depends on a Dart template', () { + final result = engine.analyze(r"SELECT 'string' = $expr;"); + final moorQuery = QueryHandler('query', result, mapper).handle(); + + expect(moorQuery.lints, + anyElement((AnalysisError q) => q.message.contains('Dart template'))); + }); +} diff --git a/sqlparser/lib/src/analysis/types/resolver.dart b/sqlparser/lib/src/analysis/types/resolver.dart index 0794bba1..4289a813 100644 --- a/sqlparser/lib/src/analysis/types/resolver.dart +++ b/sqlparser/lib/src/analysis/types/resolver.dart @@ -62,7 +62,7 @@ class TypeResolver { // todo probably needs to be nullable when coming from a join? return ResolveResult(column.type); } else if (column is ExpressionColumn) { - return resolveExpression(column.expression); + return resolveOrInfer(column.expression); } throw StateError('Unknown column $column'); From 9f8ccd08d0f3cb92988aca84483d4d0ccd081fe0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 15 Sep 2019 17:02:01 +0200 Subject: [PATCH 099/117] Actually make the ffi package work on Android --- .cirrus.yml | 4 +-- .../integration_tests/flutter_db/lib/ffi.dart | 34 +++++++++++++++++++ .../flutter_db/lib/main.dart | 20 ++--------- moor/pubspec.yaml | 2 +- moor_ffi/android/build.gradle | 3 +- moor_ffi/android/src/main/AndroidManifest.xml | 6 ++++ moor_ffi/pubspec.yaml | 9 ++--- 7 files changed, 52 insertions(+), 26 deletions(-) create mode 100644 extras/integration_tests/flutter_db/lib/ffi.dart create mode 100644 moor_ffi/android/src/main/AndroidManifest.xml diff --git a/.cirrus.yml b/.cirrus.yml index 4cb8e831..0338e46e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,6 +1,6 @@ # Run tasks with the dart SDK installed by default -container: # todo set back to latest - image: "google/dart:dev" +container: + image: "google/dart:latest" # We're currently not running tests with coverage because the free cirrus containers run out of memory :( diff --git a/extras/integration_tests/flutter_db/lib/ffi.dart b/extras/integration_tests/flutter_db/lib/ffi.dart new file mode 100644 index 00000000..b3afb3e0 --- /dev/null +++ b/extras/integration_tests/flutter_db/lib/ffi.dart @@ -0,0 +1,34 @@ +import 'dart:io'; + +import 'package:flutter/widgets.dart' show WidgetsFlutterBinding; +import 'package:moor_ffi/moor_ffi.dart'; +import 'package:tests/tests.dart'; +import 'package:moor_flutter/moor_flutter.dart'; +import 'package:sqflite/sqflite.dart' show getDatabasesPath; +import 'package:path/path.dart'; + +class FfiExecutor extends TestExecutor { + final String dbPath; + + FfiExecutor(this.dbPath); + + @override + QueryExecutor createExecutor() { + return VmDatabase(File(join(dbPath, 'app_ffi.db'))); + } + + @override + Future deleteData() async { + final file = File(join(dbPath, 'app_ffi.db')); + if (await file.exists()) { + await file.delete(); + } + } +} + +void main() async { + WidgetsFlutterBinding.ensureInitialized(); + + final dbPath = await getDatabasesPath(); + runAllTests(FfiExecutor(dbPath)); +} diff --git a/extras/integration_tests/flutter_db/lib/main.dart b/extras/integration_tests/flutter_db/lib/main.dart index 9716ce4e..18a16290 100644 --- a/extras/integration_tests/flutter_db/lib/main.dart +++ b/extras/integration_tests/flutter_db/lib/main.dart @@ -1,7 +1,7 @@ import 'dart:io'; import 'package:flutter/services.dart'; -import 'package:moor_ffi/moor_ffi.dart'; +import 'package:flutter/widgets.dart' show WidgetsFlutterBinding; import 'package:tests/tests.dart'; import 'package:test/test.dart'; import 'package:moor_flutter/moor_flutter.dart'; @@ -28,24 +28,10 @@ class SqfliteExecutor extends TestExecutor { } } -class FfiExecutor extends TestExecutor { - @override - QueryExecutor createExecutor() { - return VmDatabase(File('app_ffi.db')); - } +void main() async { + WidgetsFlutterBinding.ensureInitialized(); - @override - Future deleteData() async { - final file = File('app_ffi.db'); - if (await file.exists()) { - await file.delete(); - } - } -} - -void main() { runAllTests(SqfliteExecutor()); - runAllTests(FfiExecutor()); // Additional integration test for flutter: Test loading a database from asset test('can load a database from asset', () async { diff --git a/moor/pubspec.yaml b/moor/pubspec.yaml index 7effd858..de6c8901 100644 --- a/moor/pubspec.yaml +++ b/moor/pubspec.yaml @@ -9,7 +9,7 @@ authors: maintainer: Simon Binder (@simolus3) environment: - sdk: '>=2.5.0-dev <3.0.0' + sdk: '>=2.3.0 <3.0.0' dependencies: meta: ^1.0.0 diff --git a/moor_ffi/android/build.gradle b/moor_ffi/android/build.gradle index c37ae8bb..8f9a49bd 100644 --- a/moor_ffi/android/build.gradle +++ b/moor_ffi/android/build.gradle @@ -13,10 +13,9 @@ plugins { id "de.undercouch.download" version "4.0.0" } -group 'com.example.moor_ffi' +group 'eu.simonbinder.moor_ffi' version '1.0' - rootProject.allprojects { repositories { google() diff --git a/moor_ffi/android/src/main/AndroidManifest.xml b/moor_ffi/android/src/main/AndroidManifest.xml new file mode 100644 index 00000000..2ac4a3a3 --- /dev/null +++ b/moor_ffi/android/src/main/AndroidManifest.xml @@ -0,0 +1,6 @@ + + + \ No newline at end of file diff --git a/moor_ffi/pubspec.yaml b/moor_ffi/pubspec.yaml index 23c06087..26ba7882 100644 --- a/moor_ffi/pubspec.yaml +++ b/moor_ffi/pubspec.yaml @@ -15,10 +15,11 @@ dependencies: dev_dependencies: test: ^1.6.0 -#flutter: -# plugin: -# androidPackage: com.example.moor_ffi -# pluginClass: MoorFfiPlugin +flutter: + plugin: + # the flutter.plugin key needs to exists so that this project gets recognized as a plugin when imported. We need to + # get recognized as a plugin so that our build scripts are executed. + foo: bar dependency_overrides: moor: From 5f2d5d3258d5e3017330e0528df6642f0a3d18ea Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 15 Sep 2019 21:41:32 +0200 Subject: [PATCH 100/117] Parse IN () again --- moor_ffi/README.md | 14 +++--- sqlparser/lib/src/ast/expressions/simple.dart | 7 ++- sqlparser/lib/src/reader/parser/crud.dart | 3 +- .../lib/src/reader/parser/expressions.dart | 36 ++++++++------- sqlparser/lib/src/reader/parser/parser.dart | 5 ++- sqlparser/test/parser/expression_test.dart | 44 +++++++++++++++++++ 6 files changed, 83 insertions(+), 26 deletions(-) diff --git a/moor_ffi/README.md b/moor_ffi/README.md index 9f25f0c5..1ffbb693 100644 --- a/moor_ffi/README.md +++ b/moor_ffi/README.md @@ -1,21 +1,23 @@ # moor_ffi -Moor backend that uses the new `dart:ffi` apis. Note that, while we have integration tests +Moor backend that uses `dart:ffi`. Note that, while we have integration tests on this package, it depends on the `dart:ffi` apis, which are in "preview" status at the moment. Thus, this library is not suited for production use. If you want to use moor on Android or iOS, see the [getting started guide](https://moor.simonbinder.eu/docs/getting-started/) which recommends to use the [moor_flutter](https://pub.dev/packages/moor_flutter) package. -At the moment, this library is targeted for advanced moor users who want to try out the `ffi` +At the moment, this library is targeted at advanced moor users who want to try out the `ffi` backend. ## Supported platforms -At the moment, this plugin supports Android natively. However, it's also going to run on all -platforms that expose `sqlite3` as a shared native library (macOS and virtually all Linux -distros, I'm not sure about Windows). Native iOS and macOS support is planned. +At the moment, this plugin only supports Android without further work. However, it's also going +to run on all platforms that expose `sqlite3` as a shared native library (macOS and virtually +all Linux distros, I'm not sure about Windows). Native iOS and macOS support is planned. +As Flutter desktop doesn't support plugins on Windows and Linux yet, we can't bundle the +sqlite library on those platforms. ## Migrating from moor_flutter -Add both `moor` and `moor_ffi` to your pubspec. +Add both `moor` and `moor_ffi` to your pubspec, the `moor_flutter` dependency can be dropped. ```yaml dependencies: diff --git a/sqlparser/lib/src/ast/expressions/simple.dart b/sqlparser/lib/src/ast/expressions/simple.dart index 48bb0c81..a2b638ba 100644 --- a/sqlparser/lib/src/ast/expressions/simple.dart +++ b/sqlparser/lib/src/ast/expressions/simple.dart @@ -122,10 +122,15 @@ class BetweenExpression extends Expression { class InExpression extends Expression { final bool not; final Expression left; + + /// The right-hand part: Contains the set of values [left] will be tested + /// against. From the sqlite grammar, we support [Tuple] and a [SubQuery]. + /// We also support a [Variable] as syntax sugar - it will be expanded into a + /// tuple of variables at runtime. final Expression inside; InExpression({this.not = false, @required this.left, @required this.inside}) { - assert(inside is Tuple || inside is Variable); + assert(inside is Tuple || inside is Variable || inside is SubQuery); } @override diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 9edca466..0cf8c929 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -460,7 +460,8 @@ mixin CrudParser on ParserBase { if (_matchOne(TokenType.$values)) { final values = []; do { - values.add(_consumeTuple()); + // it will be a tuple, we don't turn on "orSubQuery" + values.add(_consumeTuple() as Tuple); } while (_matchOne(TokenType.comma)); return ValuesSource(values); } else if (_matchOne(TokenType.$default)) { diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index ff6b4fc9..e73d432c 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -67,7 +67,7 @@ mixin ExpressionParser on ParserBase { final not = _matchOne(TokenType.not); _matchOne(TokenType.$in); - final inside = _variableOrNull() ?? _consumeTuple(); + final inside = _variableOrNull() ?? _consumeTuple(orSubQuery: true); return InExpression(left: left, inside: inside, not: not); } @@ -289,12 +289,6 @@ mixin ExpressionParser on ParserBase { return Reference(columnName: first.identifier)..setSpan(first, first); } break; - case TokenType.questionMarkVariable: - return NumberedVariable(token as QuestionMarkVariableToken) - ..setSpan(token, token); - case TokenType.colonVariable: - return ColonNamedVariable(token as ColonVariableToken) - ..setSpan(token, token); case TokenType.dollarSignVariable: if (enableMoorExtensions) { final typedToken = token as DollarSignVariableToken; @@ -377,20 +371,28 @@ mixin ExpressionParser on ParserBase { } @override - Tuple _consumeTuple() { + Expression _consumeTuple({bool orSubQuery = false}) { final firstToken = _consume(TokenType.leftParen, 'Expected opening parenthesis for tuple'); final expressions = []; - // tuples can be empty `()`, so only start parsing values when it's not - if (_peek.type != TokenType.rightParen) { - do { - expressions.add(expression()); - } while (_matchOne(TokenType.comma)); + final subQuery = select(); + if (subQuery == null) { + // no sub query found. read expressions that form the tuple. + // tuples can be empty `()`, so only start parsing values when it's not + if (_peek.type != TokenType.rightParen) { + do { + expressions.add(expression()); + } while (_matchOne(TokenType.comma)); + } + + _consume( + TokenType.rightParen, 'Expected right parenthesis to close tuple'); + return Tuple(expressions: expressions)..setSpan(firstToken, _previous); + } else { + _consume(TokenType.rightParen, + 'Expected right parenthesis to finish subquery'); + return SubQuery(select: subQuery)..setSpan(firstToken, _previous); } - - _consume(TokenType.rightParen, 'Expected right parenthesis to close tuple'); - - return Tuple(expressions: expressions)..setSpan(firstToken, _previous); } } diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index 237a3e89..adbf194c 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -142,7 +142,10 @@ abstract class ParserBase { // Common operations that we are referenced very often Expression expression(); - Tuple _consumeTuple(); + + /// Parses a [Tuple]. If [orSubQuery] is set (defaults to false), a [SubQuery] + /// (in brackets) will be accepted as well. + Expression _consumeTuple({bool orSubQuery = false}); /// Parses a [SelectStatement], or returns null if there is no select token /// after the current position. diff --git a/sqlparser/test/parser/expression_test.dart b/sqlparser/test/parser/expression_test.dart index 3e62aeb8..998e1f01 100644 --- a/sqlparser/test/parser/expression_test.dart +++ b/sqlparser/test/parser/expression_test.dart @@ -83,6 +83,15 @@ final Map _testCases = { ), ), ), + '(SELECT x)': SubQuery( + select: SelectStatement( + columns: [ + ExpressionResultColumn( + expression: Reference(columnName: 'x'), + ), + ], + ), + ), "'hello' || 'world' COLLATE NOCASE": BinaryExpression( StringLiteral.from(token(TokenType.stringLiteral), 'hello'), token(TokenType.doublePipe), @@ -92,6 +101,41 @@ final Map _testCases = { collateFunction: token(TokenType.identifier), ), ), + 'x in ?': InExpression( + left: Reference(columnName: 'x'), + inside: NumberedVariable(QuestionMarkVariableToken(fakeSpan('?'), null)), + ), + 'x IN (SELECT col FROM tbl)': InExpression( + left: Reference(columnName: 'x'), + inside: SubQuery( + select: SelectStatement( + columns: [ + ExpressionResultColumn( + expression: Reference(columnName: 'col'), + ) + ], + from: [ + TableReference('tbl', null), + ], + ), + ), + ), + 'x IN (1, 2, (SELECT 3))': InExpression( + left: Reference(columnName: 'x'), + inside: Tuple( + expressions: [ + NumericLiteral(1.0, token(TokenType.numberLiteral)), + NumericLiteral(2.0, token(TokenType.numberLiteral)), + SubQuery( + select: SelectStatement(columns: [ + ExpressionResultColumn( + expression: NumericLiteral(3.0, token(TokenType.numberLiteral)), + ), + ]), + ), + ], + ), + ), }; void main() { From 2bf42a6157b0cd37947b296d2850cd95a50301da Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 16 Sep 2019 19:36:56 +0200 Subject: [PATCH 101/117] Resolve table references declared by foreign key clause --- .../lib/src/analyzer/moor/parser.dart | 11 ++- .../lib/src/analyzer/moor/table_handler.dart | 52 ++++++++++++++ .../lib/src/analyzer/runner/results.dart | 4 +- .../lib/src/analyzer/runner/steps.dart | 2 + .../analyzer/runner/steps/analyze_dart.dart | 22 +++++- .../analyzer/runner/steps/analyze_moor.dart | 3 + .../lib/src/model/specified_table.dart | 19 ++++- .../lib/src/utils/table_reference_sorter.dart | 70 +++++++++++++++++++ .../test/analyzer/integration_test.dart | 9 +++ .../utils/table_reference_sorter_test.dart | 54 ++++++++++++++ 10 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 moor_generator/lib/src/analyzer/moor/table_handler.dart create mode 100644 moor_generator/lib/src/utils/table_reference_sorter.dart create mode 100644 moor_generator/test/utils/table_reference_sorter_test.dart diff --git a/moor_generator/lib/src/analyzer/moor/parser.dart b/moor_generator/lib/src/analyzer/moor/parser.dart index e7755ecb..a0e73726 100644 --- a/moor_generator/lib/src/analyzer/moor/parser.dart +++ b/moor_generator/lib/src/analyzer/moor/parser.dart @@ -2,6 +2,7 @@ import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/analyzer/moor/create_table_reader.dart'; import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; import 'package:sqlparser/sqlparser.dart'; @@ -38,14 +39,20 @@ class MoorParser { )); } - final createdTables = - createdReaders.map((r) => r.extractTable(step.mapper)).toList(); + final createdTables = []; + final tableDeclarations = {}; + for (var reader in createdReaders) { + final table = reader.extractTable(step.mapper); + createdTables.add(table); + tableDeclarations[reader.stmt] = table; + } final analyzedFile = ParsedMoorFile( result, declaredTables: createdTables, queries: queryDeclarations, imports: importStatements, + tableDeclarations: tableDeclarations, ); for (var decl in queryDeclarations) { decl.file = analyzedFile; diff --git a/moor_generator/lib/src/analyzer/moor/table_handler.dart b/moor_generator/lib/src/analyzer/moor/table_handler.dart new file mode 100644 index 00000000..f2acac41 --- /dev/null +++ b/moor_generator/lib/src/analyzer/moor/table_handler.dart @@ -0,0 +1,52 @@ +import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/analyzer/runner/steps.dart'; +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:sqlparser/sqlparser.dart'; + +/// Handles `REFERENCES` clauses in tables by resolving their columns and +/// reporting errors if they don't exist. Further, sets the +/// [SpecifiedTable.references] field for tables declared in moor. +class TableHandler { + final AnalyzeMoorStep step; + final ParsedMoorFile file; + final List availableTables; + + TableHandler(this.step, this.file, this.availableTables); + + void handle() { + for (var table in file.declaredTables) { + table.references.clear(); + } + + file.parseResult.rootNode?.accept(_ReferenceResolvingVisitor(this)); + } +} + +class _ReferenceResolvingVisitor extends RecursiveVisitor { + final TableHandler handler; + + _ReferenceResolvingVisitor(this.handler); + + @override + void visitForeignKeyClause(ForeignKeyClause clause) { + final stmt = clause.parents.whereType().first; + final referencedTable = handler.availableTables.singleWhere( + (t) => t.sqlName == clause.foreignTable.tableName, + orElse: () => null); + + if (referencedTable == null) { + handler.step.reportError(ErrorInMoorFile( + severity: Severity.error, + span: clause.span, + message: + 'Referenced table ${clause.foreignTable.tableName} could not be' + 'found.')); + } else { + final createdTable = handler.file.tableDeclarations[stmt]; + createdTable?.references?.add(referencedTable); + } + + super.visitForeignKeyClause(clause); + } +} diff --git a/moor_generator/lib/src/analyzer/runner/results.dart b/moor_generator/lib/src/analyzer/runner/results.dart index ba64ef07..5ee8934f 100644 --- a/moor_generator/lib/src/analyzer/runner/results.dart +++ b/moor_generator/lib/src/analyzer/runner/results.dart @@ -37,11 +37,13 @@ class ParsedMoorFile extends FileResult { final List queries; List resolvedQueries; + Map tableDeclarations; Map resolvedImports; ParsedMoorFile(this.parseResult, {List declaredTables = const [], this.queries = const [], - this.imports = const []}) + this.imports = const [], + this.tableDeclarations = const {}}) : super(declaredTables); } diff --git a/moor_generator/lib/src/analyzer/runner/steps.dart b/moor_generator/lib/src/analyzer/runner/steps.dart index 4f9ca856..44ae604b 100644 --- a/moor_generator/lib/src/analyzer/runner/steps.dart +++ b/moor_generator/lib/src/analyzer/runner/steps.dart @@ -4,6 +4,7 @@ import 'package:analyzer/dart/element/type.dart'; import 'package:moor/moor.dart'; import 'package:moor_generator/src/analyzer/dart/parser.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/moor/table_handler.dart'; import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/analyzer/runner/results.dart'; import 'package:moor_generator/src/analyzer/moor/inline_dart_resolver.dart'; @@ -14,6 +15,7 @@ import 'package:moor_generator/src/analyzer/runner/task.dart'; import 'package:moor_generator/src/model/specified_db_classes.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:moor_generator/src/utils/table_reference_sorter.dart'; import 'package:source_gen/source_gen.dart'; part 'steps/analyze_dart.dart'; diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart index 8ee8fd46..ac44fdd4 100644 --- a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart @@ -10,10 +10,30 @@ class AnalyzeDartStep extends AnalyzingStep { for (var accessor in parseResult.dbAccessors) { final transitiveImports = _transitiveImports(accessor.resolvedImports); - final availableTables = _availableTables(transitiveImports) + var availableTables = _availableTables(transitiveImports) .followedBy(accessor.tables) .toList(); + try { + availableTables = sortTablesTopologically(availableTables); + } on CircularReferenceException catch (e) { + final msg = StringBuffer( + 'Found a circular reference in your database. This can cause ' + 'exceptions at runtime when opening the database. This is the ' + 'cycle that we found: '); + + msg.write(e.affected.map((t) => t.displayName).join(' -> ')); + // the last table in e.affected references the first one. Let's make + // that clear in the visualization. + msg.write(' -> ${e.affected.first.displayName}'); + + reportError(ErrorInDartCode( + severity: Severity.warning, + affectedElement: accessor.fromClass, + message: msg.toString(), + )); + } + final availableQueries = transitiveImports .map((f) => f.currentResult) .whereType() diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart index 95b9b783..823e7b9a 100644 --- a/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_moor.dart @@ -15,6 +15,9 @@ class AnalyzeMoorStep extends AnalyzingStep { final parser = SqlParser(this, availableTables, parseResult.queries) ..parse(); + + TableHandler(this, parseResult, availableTables).handle(); + parseResult.resolvedQueries = parser.foundQueries; } } diff --git a/moor_generator/lib/src/model/specified_table.dart b/moor_generator/lib/src/model/specified_table.dart index 2a99fc7c..2c249171 100644 --- a/moor_generator/lib/src/model/specified_table.dart +++ b/moor_generator/lib/src/model/specified_table.dart @@ -60,7 +60,11 @@ class SpecifiedTable { /// `customConstraints` getter in the table class with this value. final List overrideTableConstraints; - const SpecifiedTable( + /// The set of tables referenced somewhere in the declaration of this table, + /// for instance by using a `REFERENCES` column constraint. + final Set references = {}; + + SpecifiedTable( {this.fromClass, this.columns, this.sqlName, @@ -75,6 +79,19 @@ class SpecifiedTable { /// Finds all type converters used in this tables. Iterable get converters => columns.map((c) => c.typeConverter).where((t) => t != null); + + String get displayName { + if (isFromSql) { + return sqlName; + } else { + return fromClass.displayName; + } + } + + @override + String toString() { + return 'SpecifiedTable: $displayName'; + } } String _dbFieldName(String className) => ReCase(className).camelCase; diff --git a/moor_generator/lib/src/utils/table_reference_sorter.dart b/moor_generator/lib/src/utils/table_reference_sorter.dart new file mode 100644 index 00000000..7772dee3 --- /dev/null +++ b/moor_generator/lib/src/utils/table_reference_sorter.dart @@ -0,0 +1,70 @@ +import 'package:moor_generator/src/model/specified_table.dart'; + +/// Topologically sorts a list of [SpecifiedTable]s by their +/// [SpecifiedTable.references] relationship: Tables appearing first in the +/// output have to be created first so the table creation script doesn't crash +/// because of tables not existing. +/// +/// If there is a circular reference between [SpecifiedTable]s, an error will +/// be added that contains the name of the tables in question. +List sortTablesTopologically(Iterable tables) { + final run = _SortRun(); + + for (var table in tables) { + if (!run.didVisitAlready(table)) { + run.previous[table] = null; + _visit(table, run); + } + } + + return run.result; +} + +void _visit(SpecifiedTable table, _SortRun run) { + for (var reference in table.references) { + if (run.result.contains(reference)) { + // already handled, nothing to do + } else if (run.previous.containsKey(reference)) { + // that's a circular reference, report + run.throwCircularException(table, reference); + } else { + run.previous[reference] = table; + _visit(reference, run); + } + } + + // now that everything this table references is written, add the table itself + run.result.add(table); +} + +class _SortRun { + Map previous = {}; + final List result = []; + + /// Throws a [CircularReferenceException] because the [last] table depends on + /// [first], which transitively depends on [last] first. + void throwCircularException(SpecifiedTable last, SpecifiedTable first) { + final constructedPath = []; + for (var current = last; current != first; current = previous[current]) { + constructedPath.insert(0, current); + } + constructedPath.insert(0, first); + + throw CircularReferenceException(constructedPath); + } + + bool didVisitAlready(SpecifiedTable table) { + return previous[table] != null || result.contains(table); + } +} + +/// Thrown by [sortTablesTopologically] when the graph formed by +/// [SpecifiedTable]s and their [SpecifiedTable.references] is not acyclic. +class CircularReferenceException implements Exception { + /// The list of tables forming a circular reference, so that the first table + /// in this list references the second one and so on. The last table in this + /// list references the first one. + final List affected; + + CircularReferenceException(this.affected); +} diff --git a/moor_generator/test/analyzer/integration_test.dart b/moor_generator/test/analyzer/integration_test.dart index 16b18936..0aaa7c46 100644 --- a/moor_generator/test/analyzer/integration_test.dart +++ b/moor_generator/test/analyzer/integration_test.dart @@ -43,6 +43,11 @@ class Database {} AssetId.parse('test_lib|lib/tables.moor'): r''' import 'another.dart'; +CREATE TABLE reference_test ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + library INT NOT NULL REFERENCES libraries(id) +); + CREATE TABLE libraries ( id INT NOT NULL PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL @@ -90,6 +95,10 @@ class ProgrammingLanguages extends Table { expect(database.allTables.map((t) => t.sqlName), containsAll(['used_languages', 'libraries', 'programming_languages'])); + final tableWithReferences = + database.allTables.singleWhere((r) => r.sqlName == 'reference_test'); + expect(tableWithReferences.references.single.sqlName, 'libraries'); + final importQuery = database.resolvedQueries .singleWhere((q) => q.name == 'transitiveImportTest') as SqlSelectQuery; expect(importQuery.resultClassName, 'ProgrammingLanguage'); diff --git a/moor_generator/test/utils/table_reference_sorter_test.dart b/moor_generator/test/utils/table_reference_sorter_test.dart new file mode 100644 index 00000000..d57d68d1 --- /dev/null +++ b/moor_generator/test/utils/table_reference_sorter_test.dart @@ -0,0 +1,54 @@ +import 'package:moor_generator/src/model/specified_table.dart'; +import 'package:moor_generator/src/utils/table_reference_sorter.dart'; +import 'package:test/test.dart'; + +void main() { + test('throws cyclic exception when two tables reference each other', () { + final first = SpecifiedTable(sqlName: 'a'); + final second = SpecifiedTable(sqlName: 'b'); + first.references.add(second); + second.references.add(first); + + final exception = _expectFails([first, second]); + + expect(exception.affected, [first, second]); + }); + + test('throws cyclic exception on a circular reference with three tables', () { + final a = SpecifiedTable(sqlName: 'a'); + final b = SpecifiedTable(sqlName: 'b'); + final c = SpecifiedTable(sqlName: 'c'); + final d = SpecifiedTable(sqlName: 'd'); + + a.references.add(b); + b.references.add(c); + c.references.add(d); + d.references.add(b); + + final exception = _expectFails([a, b, c, d]); + + expect(exception.affected, [b, c, d]); + }); + + test('sorts tables topologically when no cycles exist', () { + final a = SpecifiedTable(sqlName: 'a'); + final b = SpecifiedTable(sqlName: 'b'); + final c = SpecifiedTable(sqlName: 'c'); + final d = SpecifiedTable(sqlName: 'd'); + + a.references.add(b); + b.references.add(c); + + final sorted = sortTablesTopologically([a, b, c, d]); + expect(sorted, [c, b, a, d]); + }); +} + +CircularReferenceException _expectFails(Iterable table) { + try { + sortTablesTopologically(table); + fail('Expected sortTablesTopologically to throw here'); + } on CircularReferenceException catch (e) { + return e; + } +} From 746b8401ec6d8b0badac54277191fa0454f7fd46 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 16 Sep 2019 22:33:36 +0200 Subject: [PATCH 102/117] Scan comments in sql --- .../lib/src/utils/table_reference_sorter.dart | 5 +- sqlparser/lib/src/engine/sql_engine.dart | 15 ++++- .../lib/src/reader/tokenizer/scanner.dart | 46 ++++++++++++- sqlparser/lib/src/reader/tokenizer/token.dart | 21 ++++++ sqlparser/test/parser/moor_file_test.dart | 67 +++++++++++++++++++ sqlparser/test/scanner/comments_test.dart | 35 ++++++++++ 6 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 sqlparser/test/parser/moor_file_test.dart create mode 100644 sqlparser/test/scanner/comments_test.dart diff --git a/moor_generator/lib/src/utils/table_reference_sorter.dart b/moor_generator/lib/src/utils/table_reference_sorter.dart index 7772dee3..1f6f8a0f 100644 --- a/moor_generator/lib/src/utils/table_reference_sorter.dart +++ b/moor_generator/lib/src/utils/table_reference_sorter.dart @@ -38,11 +38,12 @@ void _visit(SpecifiedTable table, _SortRun run) { } class _SortRun { - Map previous = {}; + final Map previous = {}; final List result = []; /// Throws a [CircularReferenceException] because the [last] table depends on - /// [first], which transitively depends on [last] first. + /// [first], which (transitively) depends on [last] as well. The path in the + /// thrown exception will go from [first] to [last]. void throwCircularException(SpecifiedTable last, SpecifiedTable first) { final constructedPath = []; for (var current = last; current != first; current = previous[current]) { diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 1146e819..6968c8af 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -35,6 +35,11 @@ class SqlEngine { /// Tokenizes the [source] into a list list [Token]s. Each [Token] contains /// information about where it appears in the [source] and a [TokenType]. + /// + /// Note that the list might be tokens that should be + /// [Token.invisibleToParser], if you're passing them to a [Parser] directly, + /// you need to filter them. When using the methods in this class, this will + /// be taken care of automatically. List tokenize(String source) { final scanner = Scanner(source, scanMoorTokens: useMoorExtensions); final tokens = scanner.scanTokens(); @@ -49,7 +54,8 @@ class SqlEngine { /// Parses the [sql] statement into an AST-representation. ParseResult parse(String sql) { final tokens = tokenize(sql); - final parser = Parser(tokens, useMoor: useMoorExtensions); + final tokensForParser = tokens.where((t) => !t.invisibleToParser).toList(); + final parser = Parser(tokensForParser, useMoor: useMoorExtensions); final stmt = parser.statement(); return ParseResult._(stmt, tokens, parser.errors, sql, null); @@ -62,7 +68,9 @@ class SqlEngine { final autoComplete = AutoCompleteEngine(); final tokens = tokenize(content); - final parser = Parser(tokens, useMoor: true, autoComplete: autoComplete); + final tokensForParser = tokens.where((t) => !t.invisibleToParser).toList(); + final parser = + Parser(tokensForParser, useMoor: true, autoComplete: autoComplete); final moorFile = parser.moorFile(); @@ -134,7 +142,8 @@ class ParseResult { /// The topmost node in the sql AST that was parsed. final AstNode rootNode; - /// The tokens that were scanned in the source file + /// The tokens that were scanned in the source file, including those that are + /// [Token.invisibleToParser]. final List tokens; /// A list of all errors that occurred during parsing. [ParsingError.toString] diff --git a/sqlparser/lib/src/reader/tokenizer/scanner.dart b/sqlparser/lib/src/reader/tokenizer/scanner.dart index db5ccc88..69c54512 100644 --- a/sqlparser/lib/src/reader/tokenizer/scanner.dart +++ b/sqlparser/lib/src/reader/tokenizer/scanner.dart @@ -61,13 +61,22 @@ class Scanner { _addToken(TokenType.plus); break; case '-': - _addToken(TokenType.minus); + if (_match('-')) { + _lineComment(); + } else { + _addToken(TokenType.minus); + } break; case '*': _addToken(TokenType.star); break; case '/': - _addToken(TokenType.slash); + if (_match('*')) { + _cStyleComment(); + } else { + _addToken(TokenType.slash); + } + break; case '%': _addToken(TokenType.percent); @@ -378,4 +387,37 @@ class Scanner { tokens.add(InlineDartToken(_currentSpan)); } } + + /// Scans a line comment after the -- has already been read. + void _lineComment() { + final contentBuilder = StringBuffer(); + while (_peek() != '\n' && !_isAtEnd) { + contentBuilder.write(_nextChar()); + } + + tokens.add(CommentToken( + CommentMode.line, contentBuilder.toString(), _currentSpan)); + } + + /// Scans a /* ... */ comment after the first /* has already been read. + /// Note that in sqlite, these comments don't have to be terminated - they + /// will be closed by the end of input without causing a parsing error. + void _cStyleComment() { + final contentBuilder = StringBuffer(); + while (!_isAtEnd) { + if (_match('*')) { + if (!_isAtEnd && _match('/')) { + break; + } else { + // write the * we otherwise forgot to write + contentBuilder.write('*'); + } + } else { + contentBuilder.write(_nextChar()); + } + } + + tokens.add(CommentToken( + CommentMode.cStyle, contentBuilder.toString(), _currentSpan)); + } } diff --git a/sqlparser/lib/src/reader/tokenizer/token.dart b/sqlparser/lib/src/reader/tokenizer/token.dart index c47ec150..c68a6da3 100644 --- a/sqlparser/lib/src/reader/tokenizer/token.dart +++ b/sqlparser/lib/src/reader/tokenizer/token.dart @@ -136,6 +136,7 @@ enum TokenType { action, semicolon, + comment, eof, /// Moor specific token, used to declare a type converters @@ -252,6 +253,10 @@ const Map moorKeywords = { class Token { final TokenType type; + /// Whether this token should be invisible to the parser. We use this for + /// comment tokens. + bool get invisibleToParser => false; + final FileSpan span; String get lexeme => span.text; @@ -348,6 +353,22 @@ class KeywordToken extends Token { } } +enum CommentMode { line, cStyle } + +/// A comment, either started with -- or with /*. +class CommentToken extends Token { + final CommentMode mode; + + /// The content of this comment, excluding the "--", "/*", "*/". + final String content; + + @override + final bool invisibleToParser = true; + + CommentToken(this.mode, this.content, FileSpan span) + : super(TokenType.comment, span); +} + class TokenizerError { final String message; final SourceLocation location; diff --git a/sqlparser/test/parser/moor_file_test.dart b/sqlparser/test/parser/moor_file_test.dart new file mode 100644 index 00000000..1a6e5c36 --- /dev/null +++ b/sqlparser/test/parser/moor_file_test.dart @@ -0,0 +1,67 @@ +import 'package:sqlparser/sqlparser.dart'; +import 'package:sqlparser/src/utils/ast_equality.dart'; +import 'package:test/test.dart'; + +const content = r''' +import 'other.dart'; +import 'another.moor'; + +CREATE TABLE tbl ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + -- this is a single-line comment + place VARCHAR REFERENCES other(location) +) + +all: SELECT /* COUNT(*), */ * FROM tbl WHERE $predicate; +'''; + +void main() { + test('parses moor files', () { + final parsed = SqlEngine(useMoorExtensions: true).parseMoorFile(content); + final file = parsed.rootNode; + + enforceEqual( + file, + MoorFile([ + ImportStatement('other.dart'), + ImportStatement('another.moor'), + CreateTableStatement( + tableName: 'tbl', + columns: [ + ColumnDefinition( + columnName: 'id', + typeName: 'INT', + constraints: [ + NotNull(null), + PrimaryKeyColumn(null, autoIncrement: true), + ], + ), + ColumnDefinition( + columnName: 'place', + typeName: 'VARCHAR', + constraints: [ + ForeignKeyColumnConstraint( + null, + ForeignKeyClause( + foreignTable: TableReference('other', null), + columnNames: [ + Reference(columnName: 'location'), + ], + ), + ), + ], + ), + ], + ), + DeclaredStatement( + 'all', + SelectStatement( + columns: [StarResultColumn(null)], + from: [TableReference('tbl', null)], + where: DartExpressionPlaceholder(name: 'predicate'), + ), + ), + ]), + ); + }); +} diff --git a/sqlparser/test/scanner/comments_test.dart b/sqlparser/test/scanner/comments_test.dart new file mode 100644 index 00000000..59f00dd7 --- /dev/null +++ b/sqlparser/test/scanner/comments_test.dart @@ -0,0 +1,35 @@ +import 'package:sqlparser/src/reader/tokenizer/scanner.dart'; +import 'package:test/test.dart'; + +import '../parser/utils.dart'; + +void main() { + test('scanns comments', () { + const sql = r''' +--line +-- line +/*c*/ +/*multi + line */ +/* not terminated'''; + + // using whereType instead of cast because of the invisible eof token + final tokens = Scanner(sql).scanTokens().whereType(); + + expect(tokens.map((t) => t.mode), [ + CommentMode.line, + CommentMode.line, + CommentMode.cStyle, + CommentMode.cStyle, + CommentMode.cStyle, + ]); + + expect(tokens.map((t) => t.content), [ + 'line', + ' line', + 'c', + 'multi\n line ', + ' not terminated', + ]); + }); +} From 00b2956b359525a0b83e4812636304cedaac19de Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Tue, 17 Sep 2019 15:23:24 +0200 Subject: [PATCH 103/117] Allow data class names to be overridden in sql Closes #147 --- moor/test/data/tables/custom_tables.g.dart | 56 +++++++++---------- moor/test/data/tables/tables.moor | 2 +- .../moor_files_integration_test.dart | 2 +- .../analyzer/moor/create_table_reader.dart | 6 +- .../lib/src/model/specified_table.dart | 10 +++- .../lib/src/ast/statements/create_table.dart | 11 +++- sqlparser/lib/src/reader/parser/schema.dart | 9 +++ sqlparser/test/parser/moor_file_test.dart | 3 +- 8 files changed, 61 insertions(+), 38 deletions(-) diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index 5dca3282..b9a41720 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -474,24 +474,24 @@ class WithConstraints extends Table final bool dontWriteConstraints = true; } -class ConfigData extends DataClass implements Insertable { +class Config extends DataClass implements Insertable { final String configKey; final String configValue; - ConfigData({@required this.configKey, this.configValue}); - factory ConfigData.fromData(Map data, GeneratedDatabase db, + Config({@required this.configKey, this.configValue}); + factory Config.fromData(Map data, GeneratedDatabase db, {String prefix}) { final effectivePrefix = prefix ?? ''; final stringType = db.typeSystem.forDartType(); - return ConfigData( + return Config( configKey: stringType .mapFromDatabaseResponse(data['${effectivePrefix}config_key']), configValue: stringType .mapFromDatabaseResponse(data['${effectivePrefix}config_value']), ); } - factory ConfigData.fromJson(Map json, + factory Config.fromJson(Map json, {ValueSerializer serializer = const ValueSerializer.defaults()}) { - return ConfigData( + return Config( configKey: serializer.fromJson(json['configKey']), configValue: serializer.fromJson(json['configValue']), ); @@ -506,7 +506,7 @@ class ConfigData extends DataClass implements Insertable { } @override - T createCompanion>(bool nullToAbsent) { + T createCompanion>(bool nullToAbsent) { return ConfigCompanion( configKey: configKey == null && nullToAbsent ? const Value.absent() @@ -517,13 +517,13 @@ class ConfigData extends DataClass implements Insertable { ) as T; } - ConfigData copyWith({String configKey, String configValue}) => ConfigData( + Config copyWith({String configKey, String configValue}) => Config( configKey: configKey ?? this.configKey, configValue: configValue ?? this.configValue, ); @override String toString() { - return (StringBuffer('ConfigData(') + return (StringBuffer('Config(') ..write('configKey: $configKey, ') ..write('configValue: $configValue') ..write(')')) @@ -535,12 +535,12 @@ class ConfigData extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is ConfigData && + (other is Config && other.configKey == configKey && other.configValue == configValue); } -class ConfigCompanion extends UpdateCompanion { +class ConfigCompanion extends UpdateCompanion { final Value configKey; final Value configValue; const ConfigCompanion({ @@ -560,10 +560,10 @@ class ConfigCompanion extends UpdateCompanion { } } -class Config extends Table with TableInfo { +class ConfigTable extends Table with TableInfo { final GeneratedDatabase _db; final String _alias; - Config(this._db, [this._alias]); + ConfigTable(this._db, [this._alias]); final VerificationMeta _configKeyMeta = const VerificationMeta('configKey'); GeneratedTextColumn _configKey; GeneratedTextColumn get configKey => _configKey ??= _constructConfigKey(); @@ -585,7 +585,7 @@ class Config extends Table with TableInfo { @override List get $columns => [configKey, configValue]; @override - Config get asDslTable => this; + ConfigTable get asDslTable => this; @override String get $tableName => _alias ?? 'config'; @override @@ -612,9 +612,9 @@ class Config extends Table with TableInfo { @override Set get $primaryKey => {configKey}; @override - ConfigData map(Map data, {String tablePrefix}) { + Config map(Map data, {String tablePrefix}) { final effectivePrefix = tablePrefix != null ? '$tablePrefix.' : null; - return ConfigData.fromData(data, _db, prefix: effectivePrefix); + return Config.fromData(data, _db, prefix: effectivePrefix); } @override @@ -630,8 +630,8 @@ class Config extends Table with TableInfo { } @override - Config createAlias(String alias) { - return Config(_db, alias); + ConfigTable createAlias(String alias) { + return ConfigTable(_db, alias); } @override @@ -808,24 +808,24 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { WithConstraints _withConstraints; WithConstraints get withConstraints => _withConstraints ??= WithConstraints(this); - Config _config; - Config get config => _config ??= Config(this); + ConfigTable _config; + ConfigTable get config => _config ??= ConfigTable(this); Mytable _mytable; Mytable get mytable => _mytable ??= Mytable(this); - ConfigData _rowToConfigData(QueryRow row) { - return ConfigData( + Config _rowToConfig(QueryRow row) { + return Config( configKey: row.readString('config_key'), configValue: row.readString('config_value'), ); } - Selectable readConfig(String var1) { + Selectable readConfig(String var1) { return customSelectQuery('SELECT * FROM config WHERE config_key = ?', variables: [Variable.withString(var1)], - readsFrom: {config}).map(_rowToConfigData); + readsFrom: {config}).map(_rowToConfig); } - Selectable readMultiple(List var1, OrderBy clause) { + Selectable readMultiple(List var1, OrderBy clause) { var $arrayStartIndex = 1; final expandedvar1 = $expandVar($arrayStartIndex, var1.length); $arrayStartIndex += var1.length; @@ -839,15 +839,15 @@ abstract class _$CustomTablesDb extends GeneratedDatabase { ], readsFrom: { config - }).map(_rowToConfigData); + }).map(_rowToConfig); } - Selectable readDynamic(Expression predicate) { + Selectable readDynamic(Expression predicate) { final generatedpredicate = $write(predicate); return customSelectQuery( 'SELECT * FROM config WHERE ${generatedpredicate.sql}', variables: [...generatedpredicate.introducedVariables], - readsFrom: {config}).map(_rowToConfigData); + readsFrom: {config}).map(_rowToConfig); } Future writeConfig(String key, String value) { diff --git a/moor/test/data/tables/tables.moor b/moor/test/data/tables/tables.moor index e4676550..b75c016d 100644 --- a/moor/test/data/tables/tables.moor +++ b/moor/test/data/tables/tables.moor @@ -18,7 +18,7 @@ CREATE TABLE with_constraints ( create table config ( config_key TEXT not null primary key, config_value TEXT -); +) AS "Config"; CREATE TABLE mytable ( someid INTEGER NOT NULL PRIMARY KEY, diff --git a/moor/test/parsed_sql/moor_files_integration_test.dart b/moor/test/parsed_sql/moor_files_integration_test.dart index 730b1963..d933925b 100644 --- a/moor/test/parsed_sql/moor_files_integration_test.dart +++ b/moor/test/parsed_sql/moor_files_integration_test.dart @@ -81,6 +81,6 @@ void main() { verify( mock.runSelect('SELECT * FROM config WHERE config_key = ?', ['key'])); - expect(parsed, ConfigData(configKey: 'key', configValue: 'value')); + expect(parsed, Config(configKey: 'key', configValue: 'value')); }); } diff --git a/moor_generator/lib/src/analyzer/moor/create_table_reader.dart b/moor_generator/lib/src/analyzer/moor/create_table_reader.dart index e9381760..2ee9bef7 100644 --- a/moor_generator/lib/src/analyzer/moor/create_table_reader.dart +++ b/moor_generator/lib/src/analyzer/moor/create_table_reader.dart @@ -76,6 +76,8 @@ class CreateTableReader { final tableName = table.name; final dartTableName = ReCase(tableName).pascalCase; + final dataClassName = stmt.overriddenDataClassName ?? + dataClassNameForClassName(dartTableName); final constraints = table.tableConstraints.map((c) => c.span.text).toList(); @@ -91,8 +93,8 @@ class CreateTableReader { fromClass: null, columns: foundColumns.values.toList(), sqlName: table.name, - dartTypeName: dataClassNameForClassName(dartTableName), - overriddenName: ReCase(tableName).pascalCase, + dartTypeName: dataClassName, + overriddenName: dartTableName, primaryKey: primaryKey, overrideWithoutRowId: table.withoutRowId ? true : null, overrideTableConstraints: constraints.isNotEmpty ? constraints : null, diff --git a/moor_generator/lib/src/model/specified_table.dart b/moor_generator/lib/src/model/specified_table.dart index 2c249171..67ce70da 100644 --- a/moor_generator/lib/src/model/specified_table.dart +++ b/moor_generator/lib/src/model/specified_table.dart @@ -35,10 +35,14 @@ class SpecifiedTable { // directly because there is no user defined parent class. // So, turn CREATE TABLE users into something called "Users" instead of // "$UsersTable". - if (_overriddenName != null) { - return _overriddenName; + final name = _overriddenName ?? tableInfoNameForTableClass(_baseName); + if (name == dartTypeName) { + // resolve clashes if the table info class has the same name as the data + // class. This can happen because the data class name can be specified by + // the user. + return '${name}Table'; } - return tableInfoNameForTableClass(_baseName); + return name; } String get updateCompanionName => _updateCompanionName(_baseName); diff --git a/sqlparser/lib/src/ast/statements/create_table.dart b/sqlparser/lib/src/ast/statements/create_table.dart index cf7b900f..55f67038 100644 --- a/sqlparser/lib/src/ast/statements/create_table.dart +++ b/sqlparser/lib/src/ast/statements/create_table.dart @@ -11,6 +11,11 @@ class CreateTableStatement extends Statement final List tableConstraints; final bool withoutRowId; + /// Specific to moor. Overrides the name of the data class used to hold a + /// result for of this table. Will be null when the moor extensions are not + /// enabled or if no name has been set. + final String overriddenDataClassName; + Token openingBracket; Token closingBracket; @@ -19,7 +24,8 @@ class CreateTableStatement extends Statement @required this.tableName, this.columns = const [], this.tableConstraints = const [], - this.withoutRowId = false}); + this.withoutRowId = false, + this.overriddenDataClassName}); @override T accept(AstVisitor visitor) => visitor.visitCreateTableStatement(this); @@ -31,6 +37,7 @@ class CreateTableStatement extends Statement bool contentEquals(CreateTableStatement other) { return other.ifNotExists == ifNotExists && other.tableName == tableName && - other.withoutRowId == withoutRowId; + other.withoutRowId == withoutRowId && + other.overriddenDataClassName == overriddenDataClassName; } } diff --git a/sqlparser/lib/src/reader/parser/schema.dart b/sqlparser/lib/src/reader/parser/schema.dart index a0aa9363..bd362987 100644 --- a/sqlparser/lib/src/reader/parser/schema.dart +++ b/sqlparser/lib/src/reader/parser/schema.dart @@ -55,12 +55,21 @@ mixin SchemaParser on ParserBase { withoutRowId = true; } + String overriddenName; + if (enableMoorExtensions && _matchOne(TokenType.as)) { + overriddenName = _consumeIdentifier( + 'Expected the name for the data class', + lenient: true) + .identifier; + } + return CreateTableStatement( ifNotExists: ifNotExists, tableName: tableIdentifier.identifier, withoutRowId: withoutRowId, columns: columns, tableConstraints: tableConstraints, + overriddenDataClassName: overriddenName, ) ..setSpan(first, _previous) ..openingBracket = leftParen diff --git a/sqlparser/test/parser/moor_file_test.dart b/sqlparser/test/parser/moor_file_test.dart index 1a6e5c36..e457cb08 100644 --- a/sqlparser/test/parser/moor_file_test.dart +++ b/sqlparser/test/parser/moor_file_test.dart @@ -10,7 +10,7 @@ CREATE TABLE tbl ( id INT NOT NULL PRIMARY KEY AUTOINCREMENT, -- this is a single-line comment place VARCHAR REFERENCES other(location) -) +) AS RowName all: SELECT /* COUNT(*), */ * FROM tbl WHERE $predicate; '''; @@ -52,6 +52,7 @@ void main() { ], ), ], + overriddenDataClassName: 'RowName', ), DeclaredStatement( 'all', From 7121bac86632a0527bfd672d7a39d69ebd59f62e Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 18 Sep 2019 20:48:44 +0200 Subject: [PATCH 104/117] More work on the analyzer plugin - finish migration to refactored generator - document how to use the analysis plugin - better error reporting without redundant messages --- extras/plugin_example/README.md | 54 +++++++----- extras/plugin_example/lib/test.moor | 15 +++- moor/tools/analyzer_plugin/bin/plugin.dart | 33 +++++++- moor/tools/analyzer_plugin/pubspec.yaml | 13 +-- moor_generator/lib/plugin.dart | 82 +++++++++++++++++++ moor_generator/lib/src/analyzer/errors.dart | 10 +++ moor_generator/lib/src/analyzer/session.dart | 13 ++- .../src/analyzer/sql_queries/sql_parser.dart | 28 +++++-- .../src/backends/plugin/backend/driver.dart | 18 +++- .../backends/plugin/backend/file_tracker.dart | 38 +++++---- .../lib/src/backends/plugin/plugin.dart | 8 +- .../plugin/services/autocomplete.dart | 4 +- .../src/backends/plugin/services/errors.dart | 8 +- .../src/backends/plugin/services/folding.dart | 4 +- .../src/backends/plugin/services/outline.dart | 33 +++++++- .../plugin/utils/ast_to_location.dart | 20 +++++ moor_generator/lib/src/model/sql_query.dart | 14 +++- .../lib/src/writer/queries/query_writer.dart | 12 +-- sqlparser/lib/src/analysis/error.dart | 2 +- sqlparser/lib/src/ast/ast.dart | 6 +- sqlparser/lib/src/reader/parser/parser.dart | 15 +++- 21 files changed, 336 insertions(+), 94 deletions(-) create mode 100644 moor_generator/lib/src/backends/plugin/utils/ast_to_location.dart diff --git a/extras/plugin_example/README.md b/extras/plugin_example/README.md index 8b3bed47..fd1f024b 100644 --- a/extras/plugin_example/README.md +++ b/extras/plugin_example/README.md @@ -1,28 +1,40 @@ Playground to test the analyzer plugin for `.moor` files. -## Playing around with this -At the moment, [DartCode](https://dartcode.org/) with version `v3.4.0-beta.2` is needed to run the -plugin. To set up the plugin, run the following steps +Currently, we support -1. Change the file `moor/tools/analyzer_plugin/pubspec.yaml` so that the `dependency_overrides` - section points to the location where you cloned this repository. This is needed because we - can't use relative paths for dependencies in analyzer plugins yet- see https://dartbug.com/35281 -2. In VS Code, change `dart.additionalAnalyzerFileExtensions` to include `moor` files: +- showing errors in moor files +- outline +- (kinda) folding + +## Setup +To use this plugin, you'll need to perform these steps once. It is assumed that you +have already cloned the `moor` repository. + +1. Clone https://github.com/simolus3/Dart-Code and checkout the + `use-custom-file-endings` branch. +2. Run `npm install` and `npm run build` to verify that everything is working. +3. Open the forked Dart-Code repo in your regular VS Code installation. +4. Press `F5` to run the plugin in another editor instance. All subsequent + steps need to be completed in that editor. +5. In the settings of that editor, change `dart.additionalAnalyzerFileExtensions` + to include `moor` files: ```json { - "dart.additionalAnalyzerFileExtensions": [ - "moor" - ] + "dart.additionalAnalyzerFileExtensions": ["moor"] } ``` - To diagnose errors with the plugin, turning on the diagnostics server by setting a - `dart.analyzerDiagnosticsPort` and enabling the instrumentation log via `dart.analyzerInstrumentationLogFile` - is recommended as well. -3. If you already had the project open, close and re-open VS Code. Otherwise, simply open this - project. -4. Type around in a `.moor` file. -5. Notice how you don't see anything (https://github.com/Dart-Code/Dart-Code/issues/1981), but - at least the plugin output appears in the instrumentation log. - -Debugging plugins is not fun. See the [docs](https://github.com/dart-lang/sdk/blob/master/pkg/analyzer_plugin/doc/tutorial/debugging.md) -on some general guidance, and good luck. Enabling the analyzer diagnostics server can help. +6. Close that editor. + +## Running +After you completed the setup, these steps will open an editor instance that runs the plugin. +1. chdir into `moor_generator` and run `lib/plugin.dart`. You can run that file from an IDE if + you need debugging capabilities, but starting it from the command line is fine. Keep that + script running. +2. Re-open the "inner" editor with the custom Dart plugin +2. Open this folder in the editor that runs the custom Dart plugin. Wait ~15s, you should start + to see some log entries in the output of step 1. As soon as they appear, the plugin is ready + to go. + +_Note_: `lib/plugin.dart` doesn't support multiple clients. Whenever you close or reload the +editor, that script needs to be restarted as well. That script should also be running before +starting the analysis server. diff --git a/extras/plugin_example/lib/test.moor b/extras/plugin_example/lib/test.moor index 70ae6457..cc561ce0 100644 --- a/extras/plugin_example/lib/test.moor +++ b/extras/plugin_example/lib/test.moor @@ -1,3 +1,14 @@ +import 'first.dart'; +import 'second.dart'; +import 'last.dart'; + CREATE TABLE playground ( - id INT NOT NULL PRIMARY KEY AUTOINCREMENT -) \ No newline at end of file + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL +); + +unknownColumn: SELECT * FROM playground WHERE bar = 'foo'; +syntaxError: SELECT 3 + FROM playground; +lints: INSERT INTO playground DEFAULT VALUES; +variables: SELECT * FROM playground WHERE id BETWEEN :foo AND :bar; +dartTemplate: SELECT * FROM playground WHERE $predicate ORDER BY $ordering; diff --git a/moor/tools/analyzer_plugin/bin/plugin.dart b/moor/tools/analyzer_plugin/bin/plugin.dart index 61a56d45..a8b95c68 100644 --- a/moor/tools/analyzer_plugin/bin/plugin.dart +++ b/moor/tools/analyzer_plugin/bin/plugin.dart @@ -1,7 +1,36 @@ +import 'dart:convert'; import 'dart:isolate'; -import 'package:moor_generator/plugin.dart'; +//import 'package:moor_generator/plugin.dart'; +import 'package:web_socket_channel/io.dart'; + +const useProxyPlugin = true; void main(List args, SendPort sendPort) { - start(args, sendPort); + PluginProxy(sendPort).start(); +// start(args, sendPort); +} + +class PluginProxy { + final SendPort sendToAnalysisServer; + + ReceivePort _receive; + IOWebSocketChannel _channel; + + PluginProxy(this.sendToAnalysisServer); + + void start() async { + _channel = IOWebSocketChannel.connect('ws://localhost:9999'); + _receive = ReceivePort(); + sendToAnalysisServer.send(_receive.sendPort); + + _receive.listen((data) { + // the server will send messages as maps, convert to json + _channel.sink.add(json.encode(data)); + }); + + _channel.stream.listen((data) { + sendToAnalysisServer.send(json.decode(data as String)); + }); + } } diff --git a/moor/tools/analyzer_plugin/pubspec.yaml b/moor/tools/analyzer_plugin/pubspec.yaml index 18ce1018..2dcabd6c 100644 --- a/moor/tools/analyzer_plugin/pubspec.yaml +++ b/moor/tools/analyzer_plugin/pubspec.yaml @@ -3,13 +3,14 @@ version: 1.0.0 description: This pubspec is a part of moor and determines the version of the moor analyzer to load dependencies: - moor_generator: +# moor_generator: + web_socket_channel: ^1.0.15 # To work on this plugin, you need to add the absolute paths here. Relative paths aren't supported yet # https://github.com/dart-lang/sdk/issues/35281 -dependency_overrides: - moor_generator: - path: /home/simon/IdeaProjects/moor/moor_generator - sqlparser: - path: /home/simon/IdeaProjects/moor/sqlparser +#dependency_overrides: +# moor_generator: +# path: /home/simon/IdeaProjects/moor/moor_generator +# sqlparser: +# path: /home/simon/IdeaProjects/moor/sqlparser diff --git a/moor_generator/lib/plugin.dart b/moor_generator/lib/plugin.dart index bb72c4b5..b0271b86 100644 --- a/moor_generator/lib/plugin.dart +++ b/moor_generator/lib/plugin.dart @@ -1,6 +1,11 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; import 'dart:isolate'; import 'package:analyzer/file_system/physical_file_system.dart'; +import 'package:analyzer_plugin/channel/channel.dart'; +import 'package:analyzer_plugin/protocol/protocol.dart'; import 'package:analyzer_plugin/starter.dart'; import 'package:moor_generator/src/backends/plugin/plugin.dart'; @@ -8,3 +13,80 @@ void start(List args, SendPort sendPort) { ServerPluginStarter(MoorPlugin(PhysicalResourceProvider.INSTANCE)) .start(sendPort); } + +class WebSocketPluginServer implements PluginCommunicationChannel { + final dynamic address; + final int port; + + HttpServer server; + + WebSocket _currentClient; + final StreamController _clientStream = + StreamController.broadcast(); + + WebSocketPluginServer({dynamic address, this.port = 9999}) + : address = address ?? InternetAddress.loopbackIPv4 { + _init(); + } + + void _init() async { + server = await HttpServer.bind(address, port); + print('listening on $address at port $port'); + server.transform(WebSocketTransformer()).listen(_handleClientAdded); + } + + void _handleClientAdded(WebSocket socket) { + if (_currentClient != null) { + print( + 'ignoring connection attempt because an active client already exists'); + socket.close(); + } else { + print('client connected'); + _currentClient = socket; + _clientStream.add(_currentClient); + _currentClient.done.then((_) { + print('client disconnected'); + _currentClient = null; + _clientStream.add(null); + }); + } + } + + @override + void close() { + server?.close(force: true); + } + + @override + void listen(void Function(Request request) onRequest, + {Function onError, void Function() onDone}) { + final stream = _clientStream.stream; + + // wait until we're connected + stream.firstWhere((socket) => socket != null).then((_) { + _currentClient.listen((data) { + print('I: $data'); + onRequest(Request.fromJson( + json.decode(data as String) as Map)); + }); + }); + stream.firstWhere((socket) => socket == null).then((_) => onDone()); + } + + @override + void sendNotification(Notification notification) { + print('N: ${notification.toJson()}'); + _currentClient?.add(json.encode(notification.toJson())); + } + + @override + void sendResponse(Response response) { + print('O: ${response.toJson()}'); + _currentClient?.add(json.encode(response.toJson())); + } +} + +/// Starts the plugin over a websocket service. +void main() { + MoorPlugin(PhysicalResourceProvider.INSTANCE).start(WebSocketPluginServer()); +} diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index 863ce41a..dae73c66 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -3,6 +3,7 @@ import 'package:collection/collection.dart'; import 'package:meta/meta.dart'; import 'package:source_gen/source_gen.dart'; import 'package:source_span/source_span.dart'; +import 'package:sqlparser/sqlparser.dart'; typedef LogFunction = void Function(dynamic message, [Object error, StackTrace stackTrace]); @@ -58,6 +59,15 @@ class ErrorInMoorFile extends MoorError { Severity severity = Severity.warning}) : super(message: message, severity: severity); + factory ErrorInMoorFile.fromSqlParser(AnalysisError error, + {Severity overrideSeverity}) { + return ErrorInMoorFile( + span: error.span, + message: error.message ?? error.type.toString(), + severity: overrideSeverity ?? Severity.error, + ); + } + @override void writeDescription(LogFunction log) { log(span.message(message)); diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 939f909a..2300a85e 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -16,13 +16,19 @@ class MoorSession { final Backend backend; final _completedTasks = StreamController.broadcast(); - final _changedFiles = StreamController.broadcast(); + final _changedFiles = StreamController>.broadcast(); MoorSession(this.backend); /// Stream that emits a [Task] that has been completed. Stream get completedTasks => _completedTasks.stream; + /// Stream that emits a list of [FoundFile] that need to be parsed or + /// re-analyzed because a file has changed. + /// This is not supported on all backends (notably, not with `package:build`, + /// which assumes immutable files during a build run). + Stream> get changedFiles => _changedFiles.stream; + FileType _findFileType(String path) { final extension = p.extension(path); @@ -53,15 +59,18 @@ class MoorSession { /// changed. void notifyFileChanged(FoundFile file) { file.state = FileState.dirty; + final changed = [file]; + // all files that transitively imported this files are no longer analyzed // because they depend on this file. They're still parsed though for (var affected in fileGraph.crawl(file, transposed: true)) { if (affected.state == FileState.analyzed) { affected.state = FileState.parsed; } + changed.add(affected); } - _changedFiles.add(file); + _changedFiles.add(changed); } void notifyTaskFinished(Task task) { diff --git a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart index c05e262d..2de0a779 100644 --- a/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart +++ b/moor_generator/lib/src/analyzer/sql_queries/sql_parser.dart @@ -1,5 +1,6 @@ import 'package:build/build.dart'; import 'package:moor_generator/src/analyzer/errors.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/analyzer/runner/steps.dart'; import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/model/sql_query.dart'; @@ -50,10 +51,9 @@ class SqlParser { } for (var error in context.errors) { - step.reportError(MoorError( - severity: Severity.warning, - message: 'The sql query $name is invalid: $error', - )); + _report(error, + msg: () => 'The sql query $name is invalid: $error', + severity: Severity.error); } try { @@ -68,11 +68,23 @@ class SqlParser { // report lints for (var query in foundQueries) { for (var lint in query.lints) { - step.reportError(MoorError( - severity: Severity.info, - message: 'Lint for ${query.name}: $lint', - )); + _report(lint, + msg: () => 'Lint for ${query.name}: $lint', + severity: Severity.warning); } } } + + void _report(AnalysisError error, + {String Function() msg, Severity severity}) { + if (step.file.type == FileType.moor) { + step.reportError( + ErrorInMoorFile.fromSqlParser(error, overrideSeverity: severity)); + } else { + step.reportError(MoorError( + severity: severity, + message: msg(), + )); + } + } } diff --git a/moor_generator/lib/src/backends/plugin/backend/driver.dart b/moor_generator/lib/src/backends/plugin/backend/driver.dart index 8981b011..772bcc7e 100644 --- a/moor_generator/lib/src/backends/plugin/backend/driver.dart +++ b/moor_generator/lib/src/backends/plugin/backend/driver.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'package:analyzer/file_system/file_system.dart'; import 'package:analyzer/src/dart/analysis/file_state.dart'; import 'package:analyzer/src/dart/analysis/driver.dart'; +import 'package:logging/logging.dart'; import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/analyzer/session.dart'; import 'package:moor_generator/src/backends/plugin/backend/file_tracker.dart'; @@ -21,6 +22,7 @@ class MoorDriver implements AnalysisDriverGeneric { final ResourceProvider _resourceProvider; /* late final */ MoorSession session; + StreamSubscription _fileChangeSubscription; bool _isWorking = false; MoorDriver(this._tracker, this._scheduler, this.dartDriver, @@ -28,12 +30,15 @@ class MoorDriver implements AnalysisDriverGeneric { _scheduler.add(this); final backend = PluginBackend(this); session = backend.session; + + _fileChangeSubscription = + session.changedFiles.listen(_tracker.notifyFilesChanged); } bool _ownsFile(String path) => path.endsWith('.moor'); FoundFile pathToFoundFile(String path) { - return session.registerFile(Uri.parse(path)); + return session.registerFile(Uri.parse('file://$path')); } @override @@ -45,6 +50,7 @@ class MoorDriver implements AnalysisDriverGeneric { @override void dispose() { + _fileChangeSubscription?.cancel(); _scheduler.remove(this); dartDriver.dispose(); _tracker.dispose(); @@ -68,10 +74,15 @@ class MoorDriver implements AnalysisDriverGeneric { try { final mostImportantFile = _tracker.fileWithHighestPriority; - final backendTask = _createTask(mostImportantFile.file.uri.path); + if (mostImportantFile.file?.isAnalyzed ?? false) { + Logger.root.fine('Blocked attempt to work on fully analyzed file'); + return; + } + final backendTask = _createTask(mostImportantFile.file.uri); final task = session.startTask(backendTask); await task.runTask(); + _tracker.handleTaskCompleted(task); } finally { _isWorking = false; } @@ -104,8 +115,7 @@ class MoorDriver implements AnalysisDriverGeneric { return source.fullName; } - PluginTask _createTask(String path) { - final uri = Uri.parse(path).replace(scheme: 'file'); + PluginTask _createTask(Uri uri) { return PluginTask(uri, this); } diff --git a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart index 17cdcc9f..1b5be3d9 100644 --- a/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart +++ b/moor_generator/lib/src/backends/plugin/backend/file_tracker.dart @@ -5,6 +5,7 @@ import 'package:analyzer/src/dart/analysis/driver.dart' show AnalysisDriverPriority; import 'package:collection/collection.dart'; import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:moor_generator/src/analyzer/runner/task.dart'; int _compareByPriority(TrackedFile a, TrackedFile b) { final aPriority = a.currentPriority?.index ?? 0; @@ -25,30 +26,19 @@ class FileTracker { _pendingWork = PriorityQueue(_compareByPriority); } - void _updateFile(TrackedFile file, Function(TrackedFile) update) { + void _notifyFilePriorityChanged(TrackedFile file) { _pendingWork.remove(file); - update(file); - // if a file is analyzed, we don't need to do anything. So don't add it to - // list of of tracked files. + // if a file is analyzed, we don't need to do anything with it. So don't add + // it back into the queue if (!file.file.isAnalyzed) { _pendingWork.add(file); } } - void _putInQueue(TrackedFile file) { - _updateFile(file, (f) { - // no action needed, insert with current priority. - }); - } - bool get hasWork => _pendingWork.isNotEmpty; TrackedFile get fileWithHighestPriority => _pendingWork.first; - void notifyAnalysisStateChanged(FoundFile file) { - _putInQueue(_addFile(file)); - } - TrackedFile _addFile(FoundFile file) { return _trackedFiles.putIfAbsent(file, () { final tracked = TrackedFile(file); @@ -57,19 +47,35 @@ class FileTracker { }); } + /// Notify the work tracker that the list of [files] has changed. It's enough + /// if any of the files in the list has changed, the others are likely + /// affected because they transitively import the changed file. This method + /// assumes that the [FoundFile.state] in each file has already been adjusted. + void notifyFilesChanged(List files) { + files.map(_addFile).forEach(_notifyFilePriorityChanged); + } + void setPriorityFiles(Iterable priority) { // remove prioritized flag from existing files for (var file in _currentPriority) { - _updateFile(file, (f) => f._prioritized = false); + file._prioritized = false; + _notifyFilePriorityChanged(file); } _currentPriority ..clear() ..addAll(priority.map(_addFile)) ..forEach((file) { - _updateFile(file, (f) => f._prioritized = true); + file._prioritized = true; + _notifyFilePriorityChanged(file); }); } + void handleTaskCompleted(Task task) { + for (var file in task.analyzedFiles) { + _notifyFilePriorityChanged(_addFile(file)); + } + } + void dispose() { _computations.close(); } diff --git a/moor_generator/lib/src/backends/plugin/plugin.dart b/moor_generator/lib/src/backends/plugin/plugin.dart index 6523c29c..2ae5d9c5 100644 --- a/moor_generator/lib/src/backends/plugin/plugin.dart +++ b/moor_generator/lib/src/backends/plugin/plugin.dart @@ -73,10 +73,10 @@ class MoorPlugin extends ServerPlugin final driver = MoorDriver(tracker, analysisDriverScheduler, dartDriver, fileContentOverlay, resourceProvider); - driver - .completedFiles() - .where((file) => file.isParsed) - .listen(errorService.handleResult); + driver.completedFiles().where((file) => file.isParsed).listen((file) { + sendNotificationsForFile(file.uri.path); + errorService.handleResult(file); + }); return driver; } diff --git a/moor_generator/lib/src/backends/plugin/services/autocomplete.dart b/moor_generator/lib/src/backends/plugin/services/autocomplete.dart index 38f4edc3..63365428 100644 --- a/moor_generator/lib/src/backends/plugin/services/autocomplete.dart +++ b/moor_generator/lib/src/backends/plugin/services/autocomplete.dart @@ -21,8 +21,8 @@ class MoorCompletingContributor implements CompletionContributor { CompletionSuggestionKind.KEYWORD, suggestion.relevance, suggestion.code, - -1, - -1, + 0, + 0, false, false, )); diff --git a/moor_generator/lib/src/backends/plugin/services/errors.dart b/moor_generator/lib/src/backends/plugin/services/errors.dart index ccbdf763..600cc772 100644 --- a/moor_generator/lib/src/backends/plugin/services/errors.dart +++ b/moor_generator/lib/src/backends/plugin/services/errors.dart @@ -4,7 +4,7 @@ import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/backends/plugin/plugin.dart'; -const _parsingErrorCode = 'moor.parsingError'; +const _genericError = 'moor.errorGeneric'; /// Sends information about errors, lints and warnings encountered in a `.moor` /// file to the analyzer. @@ -31,7 +31,7 @@ class ErrorService { final location = _findLocationForError(error, path); errors.add(AnalysisError( - severity, type, location, error.message, _parsingErrorCode)); + severity, type, location, error.message, _genericError)); } } @@ -44,9 +44,9 @@ class ErrorService { final span = error.span; final start = span.start; return Location( - path, start.offset, span.length, start.line, start.column); + path, start.offset, span.length, start.line + 1, start.column + 1); } - return Location(path, -1, -1, 0, 0); + return Location(path, 0, 0, 0, 0); } } diff --git a/moor_generator/lib/src/backends/plugin/services/folding.dart b/moor_generator/lib/src/backends/plugin/services/folding.dart index f205b078..fa35e9a5 100644 --- a/moor_generator/lib/src/backends/plugin/services/folding.dart +++ b/moor_generator/lib/src/backends/plugin/services/folding.dart @@ -27,8 +27,8 @@ class _FoldingVisitor extends RecursiveVisitor { void visitMoorFile(MoorFile e) { // construct a folding region for import statements final imports = e.imports.toList(); - if (imports.isNotEmpty) { - final first = imports.first.firstPosition; + if (imports.length > 1) { + final first = imports[1].firstPosition; final last = imports.last.lastPosition; collector.addRegion(first, last - first, FoldingKind.DIRECTIVES); diff --git a/moor_generator/lib/src/backends/plugin/services/outline.dart b/moor_generator/lib/src/backends/plugin/services/outline.dart index 05a5cef6..45c9cf70 100644 --- a/moor_generator/lib/src/backends/plugin/services/outline.dart +++ b/moor_generator/lib/src/backends/plugin/services/outline.dart @@ -1,6 +1,7 @@ import 'package:analyzer_plugin/protocol/protocol_common.dart'; import 'package:analyzer_plugin/utilities/outline/outline.dart'; import 'package:moor_generator/src/backends/plugin/services/requests.dart'; +import 'package:moor_generator/src/backends/plugin/utils/ast_to_location.dart'; import 'package:sqlparser/sqlparser.dart'; const _defaultFlags = 0; @@ -13,7 +14,7 @@ class MoorOutlineContributor implements OutlineContributor { final moorRequest = request as MoorRequest; if (moorRequest.isMoorAndParsed) { - final visitor = _OutlineVisitor(collector); + final visitor = _OutlineVisitor(moorRequest, collector); moorRequest.parsedMoor.parsedFile.accept(visitor); } @@ -21,15 +22,18 @@ class MoorOutlineContributor implements OutlineContributor { } class _OutlineVisitor extends RecursiveVisitor { + final MoorRequest request; final OutlineCollector collector; - _OutlineVisitor(this.collector); + _OutlineVisitor(this.request, this.collector); Element _startElement(ElementKind kind, String name, AstNode e) { - final element = Element(kind, name, _defaultFlags); + final element = Element(kind, name, _defaultFlags, + location: locationOfNode(request.file, e)); final offset = e.firstPosition; final length = e.lastPosition - offset; + collector.startElement(element, offset, length); return element; @@ -45,13 +49,34 @@ class _OutlineVisitor extends RecursiveVisitor { @override void visitColumnDefinition(ColumnDefinition e) { _startElement(ElementKind.FIELD, e.columnName, e)..returnType = e.typeName; + + super.visitChildren(e); + collector.endElement(); + } + + @override + void visitMoorFile(MoorFile e) { + _startElement(ElementKind.LIBRARY, request.file.shortName, e); super.visitChildren(e); collector.endElement(); } @override void visitMoorDeclaredStatement(DeclaredStatement e) { - _startElement(ElementKind.TOP_LEVEL_VARIABLE, e.name, e); + final element = _startElement(ElementKind.TOP_LEVEL_VARIABLE, e.name, e); + + // enrich information with variable types if the query has been analyzed. + final resolved = request.parsedMoor.resolvedQueries + .singleWhere((q) => q.name == e.name, orElse: () => null); + + if (resolved != null) { + final parameterBuilder = StringBuffer('('); + final vars = resolved.elements.map((e) => e.parameterType).join(', '); + parameterBuilder..write(vars)..write(')'); + + element.parameters = parameterBuilder.toString(); + } + super.visitChildren(e); collector.endElement(); } diff --git a/moor_generator/lib/src/backends/plugin/utils/ast_to_location.dart b/moor_generator/lib/src/backends/plugin/utils/ast_to_location.dart new file mode 100644 index 00000000..bf1f8fbf --- /dev/null +++ b/moor_generator/lib/src/backends/plugin/utils/ast_to_location.dart @@ -0,0 +1,20 @@ +import 'package:analyzer_plugin/protocol/protocol_common.dart'; +import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; +import 'package:sqlparser/sqlparser.dart'; + +Location locationOfNode(FoundFile file, AstNode node) { + if (!node.hasSpan) return null; + + final first = node.first.span.start; + final last = node.last.span.end; + + // in [Location], lines and columns are one-indexed, but in [SourceLocation] + // they're 0-based. + return Location( + file.uri.path, + first.offset, + last.offset - first.offset, + first.line + 1, + first.column + 1, + ); +} diff --git a/moor_generator/lib/src/model/sql_query.dart b/moor_generator/lib/src/model/sql_query.dart index a5ede39a..aadfa620 100644 --- a/moor_generator/lib/src/model/sql_query.dart +++ b/moor_generator/lib/src/model/sql_query.dart @@ -202,6 +202,9 @@ class ResultColumn { /// such as variables or Dart placeholders. abstract class FoundElement { String get dartParameterName; + + /// The type of this element on the generated method. + String get parameterType; } /// A semantic interpretation of a [Variable] in a sql statement. @@ -242,6 +245,15 @@ class FoundVariable extends FoundElement { return 'var${variable.resolvedIndex}'; } } + + @override + String get parameterType { + final innerType = dartTypeNames[type] ?? 'dynamic'; + if (isArray) { + return 'List<$innerType>'; + } + return innerType; + } } enum DartPlaceholderType { @@ -264,7 +276,7 @@ class FoundDartPlaceholder extends FoundElement { FoundDartPlaceholder(this.type, this.columnType, this.name); - /// The type of this parameter on a generated method. + @override String get parameterType { switch (type) { case DartPlaceholderType.expression: diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index b93ad713..7e657a4d 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -205,17 +205,7 @@ class QueryWriter { void _writeParameters() { final paramList = query.elements.map((e) { - if (e is FoundVariable) { - var dartType = dartTypeNames[e.type]; - if (e.isArray) { - dartType = 'List<$dartType>'; - } - return '$dartType ${e.dartParameterName}'; - } else if (e is FoundDartPlaceholder) { - return '${e.parameterType} ${e.name}'; - } - - throw AssertionError('Unknown element (not variable of placeholder)'); + return '${e.parameterType} ${e.dartParameterName}'; }).join(', '); _buffer.write(paramList); } diff --git a/sqlparser/lib/src/analysis/error.dart b/sqlparser/lib/src/analysis/error.dart index a7289f5c..c6603d5e 100644 --- a/sqlparser/lib/src/analysis/error.dart +++ b/sqlparser/lib/src/analysis/error.dart @@ -9,7 +9,7 @@ class AnalysisError { /// The relevant portion of the source code that caused this error. Some AST /// nodes don't have a span, in that case this error is going to be null. - SourceSpan get span { + FileSpan get span { final first = relevantNode?.first?.span; final last = relevantNode?.last?.span; diff --git a/sqlparser/lib/src/ast/ast.dart b/sqlparser/lib/src/ast/ast.dart index a78af758..1832f172 100644 --- a/sqlparser/lib/src/ast/ast.dart +++ b/sqlparser/lib/src/ast/ast.dart @@ -59,10 +59,14 @@ abstract class AstNode { int get lastPosition => last.span.end.offset; FileSpan get span { - if (first == null || last == null) return null; + if (!hasSpan) return null; return first.span.expand(last.span); } + /// Whether a source span has been set on this node. The span describes what + /// range in the source code is covered by this node. + bool get hasSpan => first != null && last != null; + /// Sets the [AstNode.first] and [AstNode.last] property in one go. void setSpan(Token first, Token last) { this.first = first; diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index adbf194c..932997da 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -167,6 +167,9 @@ class Parser extends ParserBase {bool useMoor = false, AutoCompleteEngine autoComplete}) : super(tokens, useMoor, autoComplete); + // todo remove this and don't be that lazy in moorFile() + var _lastStmtHadParsingError = false; + Statement statement() { final first = _peek; Statement stmt = _crud(); @@ -205,23 +208,25 @@ class Parser extends ParserBase final first = _peek; final foundComponents = []; + // (we try again if the last statement had a parsing error) + // first, parse import statements for (var stmt = _parseAsStatement(_import); - stmt != null; + stmt != null || _lastStmtHadParsingError; stmt = _parseAsStatement(_import)) { foundComponents.add(stmt); } // next, table declarations for (var stmt = _parseAsStatement(_createTable); - stmt != null; + stmt != null || _lastStmtHadParsingError; stmt = _parseAsStatement(_createTable)) { foundComponents.add(stmt); } // finally, declared statements for (var stmt = _parseAsStatement(_declaredStatement); - stmt != null; + stmt != null || _lastStmtHadParsingError; stmt = _parseAsStatement(_declaredStatement)) { foundComponents.add(stmt); } @@ -230,6 +235,8 @@ class Parser extends ParserBase _error('Expected the file to end here.'); } + foundComponents.removeWhere((c) => c == null); + final file = MoorFile(foundComponents); if (foundComponents.isNotEmpty) { file.setSpan(first, _previous); @@ -273,11 +280,13 @@ class Parser extends ParserBase /// Invokes [parser], sets the appropriate source span and attaches a /// semicolon if one exists. T _parseAsStatement(T Function() parser) { + _lastStmtHadParsingError = false; final first = _peek; T result; try { result = parser(); } on ParsingError catch (_) { + _lastStmtHadParsingError = true; // the error is added to the list errors, so ignore. We skip to the next // semicolon to parse the next statement. _synchronize(); From 1bd856e9c58b679b58e0e329747e193007f2f470 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 18 Sep 2019 20:53:49 +0200 Subject: [PATCH 105/117] Fix scanner crashing when the last line contains a comment --- sqlparser/lib/src/reader/tokenizer/scanner.dart | 2 +- sqlparser/test/scanner/comments_test.dart | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sqlparser/lib/src/reader/tokenizer/scanner.dart b/sqlparser/lib/src/reader/tokenizer/scanner.dart index 69c54512..6b2b4159 100644 --- a/sqlparser/lib/src/reader/tokenizer/scanner.dart +++ b/sqlparser/lib/src/reader/tokenizer/scanner.dart @@ -391,7 +391,7 @@ class Scanner { /// Scans a line comment after the -- has already been read. void _lineComment() { final contentBuilder = StringBuffer(); - while (_peek() != '\n' && !_isAtEnd) { + while (!_isAtEnd && _peek() != '\n') { contentBuilder.write(_nextChar()); } diff --git a/sqlparser/test/scanner/comments_test.dart b/sqlparser/test/scanner/comments_test.dart index 59f00dd7..df52a0ad 100644 --- a/sqlparser/test/scanner/comments_test.dart +++ b/sqlparser/test/scanner/comments_test.dart @@ -32,4 +32,13 @@ void main() { ' not terminated', ]); }); + + test('supports -- comments on last line', () { + const sql = '-- not much to see'; + + final tokens = Scanner(sql).scanTokens(); + expect(tokens, hasLength(2)); + expect((tokens[0] as CommentToken).content, ' not much to see'); + expect(tokens[1].type, TokenType.eof); + }); } From 1bdfa0289b6bbac5f087a69279d94f97f7b49784 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 18 Sep 2019 21:36:25 +0200 Subject: [PATCH 106/117] Make the parser set a span on each AST node. --- sqlparser/lib/src/reader/parser/crud.dart | 19 ++++++--- .../lib/src/reader/parser/expressions.dart | 40 +++++++++++++------ sqlparser/test/parser/expression_test.dart | 2 + sqlparser/test/parser/moor_file_test.dart | 3 ++ sqlparser/test/parser/utils.dart | 12 ++++++ 5 files changed, 58 insertions(+), 18 deletions(-) diff --git a/sqlparser/lib/src/reader/parser/crud.dart b/sqlparser/lib/src/reader/parser/crud.dart index 0cf8c929..e94ef18b 100644 --- a/sqlparser/lib/src/reader/parser/crud.dart +++ b/sqlparser/lib/src/reader/parser/crud.dart @@ -274,6 +274,7 @@ mixin CrudParser on ParserBase { OrderByBase _orderBy() { if (_matchOne(TokenType.order)) { + final orderToken = _previous; _consume(TokenType.by, 'Expected "BY" after "ORDER" token'); final terms = []; do { @@ -290,7 +291,7 @@ mixin CrudParser on ParserBase { ..setSpan(termPlaceholder.first, termPlaceholder.last); } - return OrderBy(terms: terms); + return OrderBy(terms: terms)..setSpan(orderToken, _previous); } return null; } @@ -307,7 +308,8 @@ mixin CrudParser on ParserBase { ..setSpan(expr.first, expr.last); } - return OrderingTerm(expression: expr, orderingMode: mode); + return OrderingTerm(expression: expr, orderingMode: mode) + ..setSpan(expr.first, _previous); } @override @@ -356,6 +358,8 @@ mixin CrudParser on ParserBase { DeleteStatement _deleteStmt() { if (!_matchOne(TokenType.delete)) return null; + final deleteToken = _previous; + _consume(TokenType.from, 'Expected a FROM here'); final table = _tableReference(); @@ -368,7 +372,8 @@ mixin CrudParser on ParserBase { where = expression(); } - return DeleteStatement(from: table, where: where); + return DeleteStatement(from: table, where: where) + ..setSpan(deleteToken, _previous); } UpdateStatement _update() { @@ -393,7 +398,8 @@ mixin CrudParser on ParserBase { _consume(TokenType.equal, 'Expected = after the column name'); final expr = expression(); - set.add(SetComponent(column: reference, expression: expr)); + set.add(SetComponent(column: reference, expression: expr) + ..setSpan(columnName, _previous)); } while (_matchOne(TokenType.comma)); final where = _where(); @@ -428,7 +434,7 @@ mixin CrudParser on ParserBase { insertMode = InsertMode.insert; } } else { - // is it wasn't an insert, it must have been a replace + // if it wasn't an insert, it must have been a replace insertMode = InsertMode.replace; } assert(insertMode != null); @@ -440,7 +446,8 @@ mixin CrudParser on ParserBase { if (_matchOne(TokenType.leftParen)) { do { final columnRef = _consumeIdentifier('Expected a column'); - targetColumns.add(Reference(columnName: columnRef.identifier)); + targetColumns.add(Reference(columnName: columnRef.identifier) + ..setSpan(columnRef, columnRef)); } while (_matchOne(TokenType.comma)); _consume(TokenType.rightParen, diff --git a/sqlparser/lib/src/reader/parser/expressions.dart b/sqlparser/lib/src/reader/parser/expressions.dart index e73d432c..bd5516f6 100644 --- a/sqlparser/lib/src/reader/parser/expressions.dart +++ b/sqlparser/lib/src/reader/parser/expressions.dart @@ -20,15 +20,20 @@ mixin ExpressionParser on ParserBase { Expression _case() { if (_matchOne(TokenType.$case)) { + final caseToken = _previous; + final base = _check(TokenType.when) ? null : _or(); final whens = []; Expression $else; while (_matchOne(TokenType.when)) { + final whenToken = _previous; + final whenExpr = _or(); _consume(TokenType.then, 'Expected THEN'); final then = _or(); - whens.add(WhenComponent(when: whenExpr, then: then)); + whens.add(WhenComponent(when: whenExpr, then: then) + ..setSpan(whenToken, _previous)); } if (_matchOne(TokenType.$else)) { @@ -36,7 +41,8 @@ mixin ExpressionParser on ParserBase { } _consume(TokenType.end, 'Expected END to finish the case operator'); - return CaseExpression(whens: whens, base: base, elseExpr: $else); + return CaseExpression(whens: whens, base: base, elseExpr: $else) + ..setSpan(caseToken, _previous); } return _or(); @@ -52,7 +58,8 @@ mixin ExpressionParser on ParserBase { while (_match(types)) { final operator = _previous; final right = higherPrecedence(); - expression = BinaryExpression(expression, operator, right); + expression = BinaryExpression(expression, operator, right) + ..setSpan(expression.first, _previous); } return expression; } @@ -68,7 +75,8 @@ mixin ExpressionParser on ParserBase { _matchOne(TokenType.$in); final inside = _variableOrNull() ?? _consumeTuple(orSubQuery: true); - return InExpression(left: left, inside: inside, not: not); + return InExpression(left: left, inside: inside, not: not) + ..setSpan(left.first, _previous); } return left; @@ -79,6 +87,7 @@ mixin ExpressionParser on ParserBase { /// expressions. Expression _equals() { var expression = _comparison(); + final first = expression.first; final ops = const [ TokenType.equal, @@ -104,15 +113,18 @@ mixin ExpressionParser on ParserBase { final upper = _comparison(); expression = BetweenExpression( - not: not, check: expression, lower: lower, upper: upper); + not: not, check: expression, lower: lower, upper: upper) + ..setSpan(first, _previous); } else if (_match(ops)) { final operator = _previous; if (operator.type == TokenType.$is) { final not = _match(const [TokenType.not]); // special case: is not expression - expression = IsExpression(not, expression, _comparison()); + expression = IsExpression(not, expression, _comparison()) + ..setSpan(first, _previous); } else { - expression = BinaryExpression(expression, operator, _comparison()); + expression = BinaryExpression(expression, operator, _comparison()) + ..setSpan(first, _previous); } } else if (_checkAnyWithNot(stringOps)) { final not = _matchOne(TokenType.not); @@ -130,7 +142,8 @@ mixin ExpressionParser on ParserBase { left: expression, operator: operator, right: right, - escape: escape); + escape: escape) + ..setSpan(first, _previous); } else { break; // no matching operator with this precedence was found } @@ -175,14 +188,17 @@ mixin ExpressionParser on ParserBase { ])) { final operator = _previous; final expression = _unary(); - return UnaryExpression(operator, expression); + return UnaryExpression(operator, expression) + ..setSpan(operator, expression.last); } else if (_matchOne(TokenType.exists)) { + final existsToken = _previous; _consume( TokenType.leftParen, 'Expected opening parenthesis after EXISTS'); final selectStmt = select(); _consume(TokenType.rightParen, 'Expected closing paranthesis to finish EXISTS expression'); - return ExistsExpression(select: selectStmt); + return ExistsExpression(select: selectStmt) + ..setSpan(existsToken, _previous); } return _postfix(); @@ -255,11 +271,11 @@ mixin ExpressionParser on ParserBase { if (_peek.type == TokenType.select) { final stmt = select(); _consume(TokenType.rightParen, 'Expected a closing bracket'); - return SubQuery(select: stmt); + return SubQuery(select: stmt)..setSpan(left, _previous); } else { final expr = expression(); _consume(TokenType.rightParen, 'Expected a closing bracket'); - return Parentheses(left, expr, token); + return Parentheses(left, expr, token)..setSpan(left, _previous); } break; case TokenType.identifier: diff --git a/sqlparser/test/parser/expression_test.dart b/sqlparser/test/parser/expression_test.dart index 998e1f01..59621e53 100644 --- a/sqlparser/test/parser/expression_test.dart +++ b/sqlparser/test/parser/expression_test.dart @@ -146,6 +146,8 @@ void main() { final tokens = scanner.scanTokens(); final parser = Parser(tokens); final expression = parser.expression(); + enforceHasSpan(expression); + enforceEqual(expression, expected); }); }); diff --git a/sqlparser/test/parser/moor_file_test.dart b/sqlparser/test/parser/moor_file_test.dart index e457cb08..5ada3236 100644 --- a/sqlparser/test/parser/moor_file_test.dart +++ b/sqlparser/test/parser/moor_file_test.dart @@ -2,6 +2,8 @@ import 'package:sqlparser/sqlparser.dart'; import 'package:sqlparser/src/utils/ast_equality.dart'; import 'package:test/test.dart'; +import 'utils.dart'; + const content = r''' import 'other.dart'; import 'another.moor'; @@ -19,6 +21,7 @@ void main() { test('parses moor files', () { final parsed = SqlEngine(useMoorExtensions: true).parseMoorFile(content); final file = parsed.rootNode; + enforceHasSpan(file); enforceEqual( file, diff --git a/sqlparser/test/parser/utils.dart b/sqlparser/test/parser/utils.dart index 440e1fc2..932eeaab 100644 --- a/sqlparser/test/parser/utils.dart +++ b/sqlparser/test/parser/utils.dart @@ -20,6 +20,7 @@ IdentifierToken identifier(String content) { void testStatement(String sql, AstNode expected, {bool moorMode = false}) { final parsed = SqlEngine(useMoorExtensions: moorMode).parse(sql).rootNode; + enforceHasSpan(parsed); enforceEqual(parsed, expected); } @@ -34,3 +35,14 @@ void testAll(Map testCases) { }); }); } + +/// The parser should make sure [AstNode.hasSpan] is true on relevant nodes. +void enforceHasSpan(AstNode node) { + final problematic = [node] + .followedBy(node.allDescendants) + .firstWhere((node) => !node.hasSpan, orElse: () => null); + + if (problematic != null) { + throw ArgumentError('Node $problematic did not have a span'); + } +} From 4e166f189c3c625d6e7fe2e7f38df03ed1ee246d Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 18 Sep 2019 21:40:19 +0200 Subject: [PATCH 107/117] Colorize errors in moor files again --- moor_generator/lib/src/analyzer/errors.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index dae73c66..a707d365 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -70,7 +70,7 @@ class ErrorInMoorFile extends MoorError { @override void writeDescription(LogFunction log) { - log(span.message(message)); + log(span.message(message, color: isError)); } } From a38e8832821490652152c33b3be98ef98b3f35df Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 20 Sep 2019 10:55:06 +0200 Subject: [PATCH 108/117] Ability to use moor_ffi on iOS --- moor_ffi/ios/moor_ffi.podspec | 10 ++++++---- moor_ffi/lib/src/load_library.dart | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/moor_ffi/ios/moor_ffi.podspec b/moor_ffi/ios/moor_ffi.podspec index cd4ea5e0..80c66141 100644 --- a/moor_ffi/ios/moor_ffi.podspec +++ b/moor_ffi/ios/moor_ffi.podspec @@ -12,10 +12,12 @@ A new flutter plugin project. s.license = { :file => '../LICENSE' } s.author = { 'Your Company' => 'email@example.com' } s.source = { :path => '.' } - s.source_files = 'Classes/**/*' - s.public_header_files = 'Classes/**/*.h' + #s.source_files = 'Classes/**/*' + #s.public_header_files = 'Classes/**/*.h' s.dependency 'Flutter' - s.ios.deployment_target = '8.0' -end + # when we run on Dart 2.6, we should use this library and also use DynamicLibrary.executable() + # s.dependency 'sqlite3' + s.ios.deployment_target = '8.0' +end \ No newline at end of file diff --git a/moor_ffi/lib/src/load_library.dart b/moor_ffi/lib/src/load_library.dart index 44df83e5..c3bdaea4 100644 --- a/moor_ffi/lib/src/load_library.dart +++ b/moor_ffi/lib/src/load_library.dart @@ -14,7 +14,9 @@ DynamicLibrary _defaultOpen() { if (Platform.isLinux || Platform.isAndroid) { return DynamicLibrary.open('libsqlite3.so'); } - if (Platform.isMacOS) { + if (Platform.isMacOS || Platform.isIOS) { + // todo when we use a dev version of Dart 2.6, we can (and should!!) use DynamicLibrary.executable() here +// return DynamicLibrary.executable(); return DynamicLibrary.open('libsqlite3.dylib'); } if (Platform.isWindows) { From 448ff108234beadc1c208f3642bc92e545d8cd2f Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 20 Sep 2019 18:53:45 +0200 Subject: [PATCH 109/117] Start documenting moor 2.0 features --- .../docs/Getting started/writing_queries.md | 19 ++- .../en/docs/Using SQL/custom_queries.md | 11 +- .../en/docs/Using SQL/custom_tables.md | 42 ----- docs/content/en/docs/Using SQL/moor_files.md | 148 ++++++++++++++++++ docs/content/en/docs/transactions.md | 17 +- 5 files changed, 177 insertions(+), 60 deletions(-) delete mode 100644 docs/content/en/docs/Using SQL/custom_tables.md create mode 100644 docs/content/en/docs/Using SQL/moor_files.md diff --git a/docs/content/en/docs/Getting started/writing_queries.md b/docs/content/en/docs/Getting started/writing_queries.md index 7a19c59f..48eeccb7 100644 --- a/docs/content/en/docs/Getting started/writing_queries.md +++ b/docs/content/en/docs/Getting started/writing_queries.md @@ -56,6 +56,20 @@ Future> sortEntriesAlphabetically() { ``` You can also reverse the order by setting the `mode` property of the `OrderingTerm` to `OrderingMode.desc`. + +### Single values +If you now a query is never going to return more than one row, wrapping the result in a `List` +can be tedious. Moor lets you work around that with `getSingle` and `watchSingle`: +```dart +Stream entryById(int id) { + return (select(todos)..where((t) => t.id.equals(id))).watchSingle(); +} +``` +If an entry with the provided id exists, it will be sent to the stream. Otherwise, +`null` will be added to stream. If a query used with `watchSingle` ever returns +more than one entry (which is impossible in this case), an error will be added +instead. + ## Updates and deletes You can use the generated classes to update individual fields of any row: ```dart @@ -125,4 +139,7 @@ addTodoEntry( ``` If a column is nullable or has a default value (this includes auto-increments), the field can be omitted. All other fields must be set and non-null. The `insert` method will throw -otherwise. \ No newline at end of file +otherwise. + +Multiple inserts can be batched by using `insertAll` - it takes a list of companions instead +of a single companion. \ No newline at end of file diff --git a/docs/content/en/docs/Using SQL/custom_queries.md b/docs/content/en/docs/Using SQL/custom_queries.md index d918ffc5..13768b3b 100644 --- a/docs/content/en/docs/Using SQL/custom_queries.md +++ b/docs/content/en/docs/Using SQL/custom_queries.md @@ -1,11 +1,18 @@ --- -title: "Custom queries" +title: "(Legacy) Custom queries" weight: 10 description: Let moor generate Dart from your SQL statements aliases: - /queries/custom --- +{{% alert title="Outdated feature" color="warning" %}} +With moor 2.0, we moved the new `.moor` files out of preview and added some powerful features to them. +They are easier to use than the approaches described here. While these features will continue to +be supported, moor files will get better tooling support in the future and we recommend to +migrate. See [their api]({{%relref "moor_files.md"%}}) for details. +{{% /alert %}} + Altough moor includes a fluent api that can be used to model most statements, advanced features like `GROUP BY` statements or window functions are not yet supported. You can use these features with custom statements. You don't have to miss out on other benefits @@ -44,7 +51,7 @@ To use this feature, it's helpful to know how Dart tables are named in sql. For override `tableName`, the name in sql will be the `snake_case` of the class name. So a Dart table called `Categories` will be named `categories`, a table called `UserAddressInformation` would be called `user_address_information`. The same rule applies to column getters without an explicit name. -Tables and columns declared in [SQL tables]({{< relref "custom_tables.md" >}}) will always have the +Tables and columns declared in [Moor files]({{< relref "moor_files.md" >}}) will always have the name you specified. {{% /alert %}} diff --git a/docs/content/en/docs/Using SQL/custom_tables.md b/docs/content/en/docs/Using SQL/custom_tables.md deleted file mode 100644 index 6b352798..00000000 --- a/docs/content/en/docs/Using SQL/custom_tables.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: "Tables from SQL" -weight: 20 -description: Generate tables from `CREATE TABLE` statements. ---- - -{{% alert title="Experimental feature" %}} -At the moment, creating table classes from `CREATE TABLE` statements is an experimental feature. -If you run into any issues, please create an issue and let us know, thanks! -{{% /alert %}} - -With moor, you can specify your table classes in Dart and it will generate matching -`CREATE TABLE` statements for you. But if you prefer to write `CREATE TABLE` statements and have -moor generating fitting Dart classes, that works too. - -To use this feature, create a (or multiple) `.moor` file somewhere in your project. You can fill -them with create table statements: -```sql - CREATE TABLE states ( - id INT NOT NULL PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL - ); - - CREATE TABLE experiments ( - id INT NOT NULL PRIMARY KEY AUTOINCREMENT, - description TEXT NOT NULL, - state INT REFERENCES states(id) ON UPDATE CASCADE ON DELETE SET NULL - ) -``` - -Then, import these tables to your database with: -```dart -@UseMoor(include: {'experiments.moor'}) -class ExperimentsDb extends _$ExperimentsDb { -``` - -All the tables will then be available inside your database class, just like they -would be if you wrote them in Dart. If you want to use this feature on an DAO, -you'll also need to `include` the .moor file on that class. Moor supports both -relative imports (like above) and absolute imports (like `package:your_app/src/tables/experiments.moor`) -Of course, this feature works perfectly together with features like generated -custom queries and query-streams. \ No newline at end of file diff --git a/docs/content/en/docs/Using SQL/moor_files.md b/docs/content/en/docs/Using SQL/moor_files.md new file mode 100644 index 00000000..11b58eeb --- /dev/null +++ b/docs/content/en/docs/Using SQL/moor_files.md @@ -0,0 +1,148 @@ +--- +title: "Moor files" +weight: 1 +description: Learn everything about the new `.moor` files which can contain tables and queries + +aliases: + - /docs/using-sql/custom_tables/ # Redirect from outdated "custom tables" page which has been deleted +--- + +Moor files are a new feature that lets you write all your database code in SQL - moor will generate typesafe APIs for them. + +## Getting started +To use this feature, lets create two files: `database.dart` and `tables.moor`. The Dart file is pretty straightforward: +```dart +import 'package:moor/moor.dart'; + +part 'database.g.dart'; + +@UseMoor( + include: {'tables.moor'}, +) +class MoorDb extends _$MoorDb { + MoorDb() : super(FlutterQueryExecutor.inDatabaseFolder('app.db')); + + @override + int get schemaVersion => 1; +} +``` + +We can now declare tables and queries in the moor file: +```sql +CREATE TABLE todos ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + content TEXT NOT NULL, + category INTEGER REFERENCES categories(id) +); + +CREATE TABLE categories ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + description TEXT NOT NULL +) AS Category; -- the AS xyz after the table defines the data class name + +-- we can put named sql queries in here as well: +createEntry: INSERT INTO todos (title, content) VALUES (:title, :content); +deleteById: DELETE FROM todos WHERE id = :id; +watchAllTodos: SELECT * FROM todos; +``` + +After running the build runner, moor will write the `database.g.dart` +file which contains the `_$MoorDb` superclass. Let's take a look at +what we got: + +- Generated data classes (`Todo` and `Category`), and companion versions + for inserts (see [Dart Interop](#dart-interop) for info). By default, + we strip a trailing "s" from the table name for the class. That's why + we used `AS Category` on the second table - it would have been called + `Categorie` otherwise. +- Methods to run the queries: + - a `Future createEntry(String title, String content)` method. It + creates a new todo entry with the provided data and returns the id of + the entry created. + - `Future deleteById(int id)`: Deletes a todo entry by its id, and + returns the amount of rows affected. + - `Selectable allTodos()`. It can be used to get, or + watch, all todo entries. It can be used with `allTodos().get()` and + `allTodos().watch()`. +- Classes for select statements that don't match a table. In the example + above, thats the `AllTodosResult` class, which contains all fields from + `todos` and the description of the associated category. + +## Variables +We support regular variables (`?`), explictly indexed variables (`?123`) +and colon-named variables (`:id`). We don't support variables declared +with @ or $. The compiler will attempt to infer the variable's type by +looking at its context. This lets moor generate typesafe apis for your +queries, the variables will be written as parameters to your method. + +### Arrays +If you want to check whether a value is in an array of values, you can +use `IN ?`. That's not valid sql, but moor will desugar that at runtime. So, for this query: +```sql +entriesWithId: SELECT * FROM todos WHERE id IN ?; +``` +Moor will generate a `Selectable entriesWithId(List ids)` +method (`entriesWithId([1,2])` would run `SELECT * ... id IN (?1, ?2)` +and bind the arguments accordingly). To support this, we only have two +restrictions: + +1. __No explicit variables__: Running `WHERE id IN ?2` will be rejected +at build time. As the variable is expanded, giving it a single index is +invalid. +2. __No higher explicit index after a variable__: Running +`WHERE id IN ? OR title = ?2` will also be rejected. Expanding the +variable can clash with the explicit index, which is why moor forbids +it. Of course, `id IN ? OR title = ?` will work as expected. + +## Imports +{{% alert title="Limited support" %}} +> Importing a moor file from another moor file will work as expected. + Unfortunately, importing a Dart file from moor does not work in all + scenarios. Please upvote [this issue](https://github.com/dart-lang/build/issues/493) + on the build package to help solve this. +{{% /alert %}} + +You can put import statements at the top of a `moor` file: +```sql +import 'other.moor'; -- single quotes are required for imports +``` +All tables reachable from the other file will then also be visible in +the current file and to the database that `includes` it. Importing +Dart files into a moor file will also work - then, all the tables +declared via Dart tables can be used inside queries. + +## Dart interop +Moor files work perfectly together with moor's existing Dart API: + +- you can write Dart queries for moor files: +```dart +Future insert(TodosCompanion companion) async { + await into(todos).insert(companion); +} +``` +- by importing Dart files into a moor file, you can write sql queries for + tables declared in Dart. +- generated methods for queries can be used in transactions, they work + together with auto-updating queries, etc. + +You can make most of both SQL and Dart by "Dart Templates", which is a +Dart expression that gets inlined to a query. To use them, declare a +$-variable in a query: +```sql +_filterTodos: SELECT * FROM todos WHERE $predicate; +``` +Moor will generate a `Selectable _filterTodos(Expression predicate)` method which can be used to construct dynamic +filters at runtime: +```dart +Stream> watchInCategory(int category) { + return _filterTodos(todos.category.equals(category)).watch(); +} +``` +This feature works for + +- expressions +- single ordering terms: `SELECT * FROM todos ORDER BY $term, id ASC` + will generate a method taking an `OrderingTerm`. +- whole order-by clauses: `SELECT * FROM todos ORDER BY $order` +- limit clauses: `SELECT * FROM todos LIMIT $limit` \ No newline at end of file diff --git a/docs/content/en/docs/transactions.md b/docs/content/en/docs/transactions.md index 527ecdab..70a5e297 100644 --- a/docs/content/en/docs/transactions.md +++ b/docs/content/en/docs/transactions.md @@ -16,7 +16,7 @@ following example, which deals with deleting a category, we move all todo entrie in that category back to the default category: ```dart Future deleteCategory(Category category) { - return transaction((_) async { + return transaction(() async { // first, move the affected todo entries back to the default category await customUpdate( 'UPDATE todos SET category = NULL WHERE category = ?', @@ -30,22 +30,9 @@ Future deleteCategory(Category category) { } ``` -{{% alert title="About that _" color="info" %}} -You might have noticed that `_` parameter on the `transaction()` callback. That parameter would -be a special version of the database that runs all the methods called on it inside the transaction. -In previous moor versions, it was important to call everything on that parameter, e.g. -```dart -transaction((db) async { - await db.delete(categories).delete(category); -}); -``` -Starting from moor 1.6, this is no longer neccessary, we can figure out that you meant to run that -in a transaction because it was called from inside a `transaction` callback. We're going to remove -that parameter entirely in moor 2.0. -{{% /alert %}} - ## āš ļø Gotchas There are a couple of things that should be kept in mind when working with transactions: + 1. __Await all calls__: All queries inside the transaction must be `await`-ed. The transaction will complete when the inner method completes. Without `await`, some queries might be operating on the transaction after it has been closed! This can cause data loss or runtime crashes. From 161f7c0203c216a1a06a049cf12377ed0bdfb3f5 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 20 Sep 2019 19:31:36 +0200 Subject: [PATCH 110/117] Optionally override hashCode and equals in result sets --- .../docs/Advanced Features/builder_options.md | 5 +- extras/plugin_example/README.md | 1 + extras/plugin_example/analysis_options.yaml | 6 +- moor/build.yaml | 2 +- moor/example/example.g.dart | 34 +++++++---- moor/test/data/tables/custom_tables.g.dart | 20 ++++--- moor/test/data/tables/todos.g.dart | 60 +++++++++++++------ .../build/generators/moor_generator.dart | 5 +- .../lib/src/backends/build/options.dart | 13 +++- .../lib/src/writer/queries/query_writer.dart | 4 +- .../src/writer/queries/result_set_writer.dart | 28 +++++++-- .../src/writer/tables/data_class_writer.dart | 24 ++------ .../lib/src/writer/utils/hash_code.dart | 2 + .../lib/src/writer/utils/override_equals.dart | 18 ++++++ .../test/writer/utils/hash_code_test.dart | 6 +- .../writer/utils/override_equals_test.dart | 25 ++++++++ 16 files changed, 177 insertions(+), 76 deletions(-) create mode 100644 moor_generator/lib/src/writer/utils/override_equals.dart create mode 100644 moor_generator/test/writer/utils/override_equals_test.dart diff --git a/docs/content/en/docs/Advanced Features/builder_options.md b/docs/content/en/docs/Advanced Features/builder_options.md index 65b3a641..965b7146 100644 --- a/docs/content/en/docs/Advanced Features/builder_options.md +++ b/docs/content/en/docs/Advanced Features/builder_options.md @@ -25,4 +25,7 @@ At the moment, moor supports these options: * `write_from_json_string_constructor`: boolean. Adds a `.fromJsonString` factory constructor to generated data classes. By default, we only write a `.fromJson` - constructor that takes a `Map`. \ No newline at end of file + constructor that takes a `Map`. +* `overrride_hash_and_equals_in_result_sets`: boolean. When moor generates another class + to hold the result of generated select queries, this flag controls whether moor should + override `operator ==` and `hashCode` in those classes. \ No newline at end of file diff --git a/extras/plugin_example/README.md b/extras/plugin_example/README.md index fd1f024b..8a7af6ae 100644 --- a/extras/plugin_example/README.md +++ b/extras/plugin_example/README.md @@ -24,6 +24,7 @@ have already cloned the `moor` repository. } ``` 6. Close that editor. +7. Uncomment the plugin lines in `analysis_options.yaml` ## Running After you completed the setup, these steps will open an editor instance that runs the plugin. diff --git a/extras/plugin_example/analysis_options.yaml b/extras/plugin_example/analysis_options.yaml index 9ec18836..7c7ad7ff 100644 --- a/extras/plugin_example/analysis_options.yaml +++ b/extras/plugin_example/analysis_options.yaml @@ -1,5 +1,5 @@ include: package:pedantic/analysis_options.yaml -analyzer: - plugins: - - moor \ No newline at end of file +#analyzer: +# plugins: +# - moor \ No newline at end of file diff --git a/moor/build.yaml b/moor/build.yaml index bb8a25f2..ac7bc057 100644 --- a/moor/build.yaml +++ b/moor/build.yaml @@ -3,4 +3,4 @@ targets: builders: moor_generator: options: - generate_private_watch_methods: true \ No newline at end of file + overrride_hash_and_equals_in_result_sets: true diff --git a/moor/example/example.g.dart b/moor/example/example.g.dart index b3097fa2..5e5434d0 100644 --- a/moor/example/example.g.dart +++ b/moor/example/example.g.dart @@ -6,7 +6,7 @@ part of 'example.dart'; // MoorGenerator // ************************************************************************** -// ignore_for_file: unnecessary_brace_in_string_interps +// ignore_for_file: unnecessary_brace_in_string_interps, unnecessary_this class Category extends DataClass implements Insertable { final int id; final String description; @@ -66,7 +66,9 @@ class Category extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is Category && other.id == id && other.description == description); + (other is Category && + other.id == this.id && + other.description == this.description); } class CategoriesCompanion extends UpdateCompanion { @@ -253,10 +255,10 @@ class Recipe extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is Recipe && - other.id == id && - other.title == title && - other.instructions == instructions && - other.category == category); + other.id == this.id && + other.title == this.title && + other.instructions == this.instructions && + other.category == this.category); } class RecipesCompanion extends UpdateCompanion { @@ -479,9 +481,9 @@ class Ingredient extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is Ingredient && - other.id == id && - other.name == name && - other.caloriesPer100g == caloriesPer100g); + other.id == this.id && + other.name == this.name && + other.caloriesPer100g == this.caloriesPer100g); } class IngredientsCompanion extends UpdateCompanion { @@ -691,9 +693,9 @@ class IngredientInRecipe extends DataClass bool operator ==(other) => identical(this, other) || (other is IngredientInRecipe && - other.recipe == recipe && - other.ingredient == ingredient && - other.amountInGrams == amountInGrams); + other.recipe == this.recipe && + other.ingredient == this.ingredient && + other.amountInGrams == this.amountInGrams); } class IngredientInRecipesCompanion extends UpdateCompanion { @@ -874,4 +876,12 @@ class TotalWeightResult { this.title, this.totalWeight, }); + @override + int get hashCode => $mrjf($mrjc(title.hashCode, totalWeight.hashCode)); + @override + bool operator ==(other) => + identical(this, other) || + (other is TotalWeightResult && + other.title == this.title && + other.totalWeight == this.totalWeight); } diff --git a/moor/test/data/tables/custom_tables.g.dart b/moor/test/data/tables/custom_tables.g.dart index b9a41720..fe74ad23 100644 --- a/moor/test/data/tables/custom_tables.g.dart +++ b/moor/test/data/tables/custom_tables.g.dart @@ -6,7 +6,7 @@ part of 'custom_tables.dart'; // MoorGenerator // ************************************************************************** -// ignore_for_file: unnecessary_brace_in_string_interps +// ignore_for_file: unnecessary_brace_in_string_interps, unnecessary_this class NoId extends DataClass implements Insertable { final Uint8List payload; NoId({@required this.payload}); @@ -55,7 +55,8 @@ class NoId extends DataClass implements Insertable { int get hashCode => $mrjf(payload.hashCode); @override bool operator ==(other) => - identical(this, other) || (other is NoId && other.payload == payload); + identical(this, other) || + (other is NoId && other.payload == this.payload); } class NoIdsCompanion extends UpdateCompanion { @@ -190,7 +191,7 @@ class WithDefault extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is WithDefault && other.a == a && other.b == b); + (other is WithDefault && other.a == this.a && other.b == this.b); } class WithDefaultsCompanion extends UpdateCompanion { @@ -354,7 +355,10 @@ class WithConstraint extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is WithConstraint && other.a == a && other.b == b && other.c == c); + (other is WithConstraint && + other.a == this.a && + other.b == this.b && + other.c == this.c); } class WithConstraintsCompanion extends UpdateCompanion { @@ -536,8 +540,8 @@ class Config extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is Config && - other.configKey == configKey && - other.configValue == configValue); + other.configKey == this.configKey && + other.configValue == this.configValue); } class ConfigCompanion extends UpdateCompanion { @@ -699,8 +703,8 @@ class MytableData extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is MytableData && - other.someid == someid && - other.sometext == sometext); + other.someid == this.someid && + other.sometext == this.sometext); } class MytableCompanion extends UpdateCompanion { diff --git a/moor/test/data/tables/todos.g.dart b/moor/test/data/tables/todos.g.dart index 53082979..9b1233ba 100644 --- a/moor/test/data/tables/todos.g.dart +++ b/moor/test/data/tables/todos.g.dart @@ -6,7 +6,7 @@ part of 'todos.dart'; // MoorGenerator // ************************************************************************** -// ignore_for_file: unnecessary_brace_in_string_interps +// ignore_for_file: unnecessary_brace_in_string_interps, unnecessary_this class TodoEntry extends DataClass implements Insertable { final int id; final String title; @@ -113,11 +113,11 @@ class TodoEntry extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is TodoEntry && - other.id == id && - other.title == title && - other.content == content && - other.targetDate == targetDate && - other.category == category); + other.id == this.id && + other.title == this.title && + other.content == this.content && + other.targetDate == this.targetDate && + other.category == this.category); } class TodosTableCompanion extends UpdateCompanion { @@ -355,7 +355,9 @@ class Category extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is Category && other.id == id && other.description == description); + (other is Category && + other.id == this.id && + other.description == this.description); } class CategoriesCompanion extends UpdateCompanion { @@ -560,11 +562,11 @@ class User extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is User && - other.id == id && - other.name == name && - other.isAwesome == isAwesome && - other.profilePicture == profilePicture && - other.creationTime == creationTime); + other.id == this.id && + other.name == this.name && + other.isAwesome == this.isAwesome && + other.profilePicture == this.profilePicture && + other.creationTime == this.creationTime); } class UsersCompanion extends UpdateCompanion { @@ -801,7 +803,9 @@ class SharedTodo extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is SharedTodo && other.todo == todo && other.user == user); + (other is SharedTodo && + other.todo == this.todo && + other.user == this.user); } class SharedTodosCompanion extends UpdateCompanion { @@ -988,9 +992,9 @@ class TableWithoutPKData extends DataClass bool operator ==(other) => identical(this, other) || (other is TableWithoutPKData && - other.notReallyAnId == notReallyAnId && - other.someFloat == someFloat && - other.custom == custom); + other.notReallyAnId == this.notReallyAnId && + other.someFloat == this.someFloat && + other.custom == this.custom); } class TableWithoutPKCompanion extends UpdateCompanion { @@ -1183,7 +1187,7 @@ class PureDefault extends DataClass implements Insertable { @override bool operator ==(other) => identical(this, other) || - (other is PureDefault && other.id == id && other.txt == txt); + (other is PureDefault && other.id == this.id && other.txt == this.txt); } class PureDefaultsCompanion extends UpdateCompanion { @@ -1433,6 +1437,28 @@ class AllTodosWithCategoryResult { this.catId, this.catDesc, }); + @override + int get hashCode => $mrjf($mrjc( + id.hashCode, + $mrjc( + title.hashCode, + $mrjc( + content.hashCode, + $mrjc( + targetDate.hashCode, + $mrjc(category.hashCode, + $mrjc(catId.hashCode, catDesc.hashCode))))))); + @override + bool operator ==(other) => + identical(this, other) || + (other is AllTodosWithCategoryResult && + other.id == this.id && + other.title == this.title && + other.content == this.content && + other.targetDate == this.targetDate && + other.category == this.category && + other.catId == this.catId && + other.catDesc == this.catDesc); } // ************************************************************************** diff --git a/moor_generator/lib/src/backends/build/generators/moor_generator.dart b/moor_generator/lib/src/backends/build/generators/moor_generator.dart index 6f08ce18..6bd790b7 100644 --- a/moor_generator/lib/src/backends/build/generators/moor_generator.dart +++ b/moor_generator/lib/src/backends/build/generators/moor_generator.dart @@ -13,9 +13,8 @@ class MoorGenerator extends Generator implements BaseGenerator { final writer = builder.createWriter(); if (parsed.declaredDatabases.isNotEmpty) { - writer - .leaf() - .write('// ignore_for_file: unnecessary_brace_in_string_interps\n'); + writer.leaf().write( + '// ignore_for_file: unnecessary_brace_in_string_interps, unnecessary_this\n'); } for (var db in parsed.declaredDatabases) { diff --git a/moor_generator/lib/src/backends/build/options.dart b/moor_generator/lib/src/backends/build/options.dart index 581cc7f1..3bac7b11 100644 --- a/moor_generator/lib/src/backends/build/options.dart +++ b/moor_generator/lib/src/backends/build/options.dart @@ -2,15 +2,22 @@ part of 'moor_builder.dart'; class MoorOptions { final bool generateFromJsonStringConstructor; + final bool overrideHashAndEqualsInResultSets; - MoorOptions(this.generateFromJsonStringConstructor); + MoorOptions(this.generateFromJsonStringConstructor, + this.overrideHashAndEqualsInResultSets); factory MoorOptions.fromBuilder(Map config) { final writeFromString = config['write_from_json_string_constructor'] as bool ?? false; - return MoorOptions(writeFromString); + final overrideInResultSets = + config['overrride_hash_and_equals_in_result_sets'] as bool ?? false; + + return MoorOptions(writeFromString, overrideInResultSets); } - const MoorOptions.defaults() : generateFromJsonStringConstructor = false; + const MoorOptions.defaults() + : generateFromJsonStringConstructor = false, + overrideHashAndEqualsInResultSets = false; } diff --git a/moor_generator/lib/src/writer/queries/query_writer.dart b/moor_generator/lib/src/writer/queries/query_writer.dart index 7e657a4d..e2ce4200 100644 --- a/moor_generator/lib/src/writer/queries/query_writer.dart +++ b/moor_generator/lib/src/writer/queries/query_writer.dart @@ -52,8 +52,8 @@ class QueryWriter { if (query is SqlSelectQuery) { final select = query as SqlSelectQuery; if (select.resultSet.needsOwnClass) { - final buffer = scope.findScopeOfLevel(DartScope.library).leaf(); - ResultSetWriter(select).write(buffer); + final resultSetScope = scope.findScopeOfLevel(DartScope.library); + ResultSetWriter(select, resultSetScope).write(); } _writeSelect(); } else if (query is UpdatingQuery) { diff --git a/moor_generator/lib/src/writer/queries/result_set_writer.dart b/moor_generator/lib/src/writer/queries/result_set_writer.dart index 3399a6b6..2d0deabb 100644 --- a/moor_generator/lib/src/writer/queries/result_set_writer.dart +++ b/moor_generator/lib/src/writer/queries/result_set_writer.dart @@ -1,13 +1,20 @@ import 'package:moor_generator/src/model/sql_query.dart'; +import 'package:moor_generator/src/writer/utils/hash_code.dart'; +import 'package:moor_generator/src/writer/utils/override_equals.dart'; +import 'package:moor_generator/src/writer/writer.dart'; /// Writes a class holding the result of an sql query into Dart. class ResultSetWriter { final SqlSelectQuery query; + final Scope scope; - ResultSetWriter(this.query); + ResultSetWriter(this.query, this.scope); - void write(StringBuffer into) { + void write() { final className = query.resultClassName; + final columnNames = + query.resultSet.columns.map(query.resultSet.dartNameFor).toList(); + final into = scope.leaf(); into.write('class $className {\n'); // write fields @@ -19,9 +26,20 @@ class ResultSetWriter { // write the constructor into.write('$className({'); - for (var column in query.resultSet.columns) { - into.write('this.${query.resultSet.dartNameFor(column)},'); + for (var column in columnNames) { + into.write('this.$column,'); } - into.write('});\n}\n'); + into.write('});\n'); + + // if requested, override hashCode and equals + if (scope.writer.options.overrideHashAndEqualsInResultSets) { + into.write('@override int get hashCode => '); + const HashCodeWriter().writeHashCode(columnNames, into); + into.write(';\n'); + + overrideEquals(columnNames, className, into); + } + + into.write('}\n'); } } diff --git a/moor_generator/lib/src/writer/tables/data_class_writer.dart b/moor_generator/lib/src/writer/tables/data_class_writer.dart index e0232bec..128c9bc0 100644 --- a/moor_generator/lib/src/writer/tables/data_class_writer.dart +++ b/moor_generator/lib/src/writer/tables/data_class_writer.dart @@ -1,5 +1,6 @@ import 'package:moor_generator/src/model/specified_table.dart'; import 'package:moor_generator/src/writer/utils/hash_code.dart'; +import 'package:moor_generator/src/writer/utils/override_equals.dart'; import 'package:moor_generator/src/writer/writer.dart'; import 'package:recase/recase.dart'; @@ -50,24 +51,11 @@ class DataClassWriter { _writeToString(); _writeHashCode(); - // override == - // return identical(this, other) || (other is DataClass && other.id == id && ...) - _buffer - ..write('@override\nbool operator ==(other) => ') - ..write('identical(this, other) || (other is ${table.dartTypeName}'); + overrideEquals(table.columns.map((c) => c.dartGetterName), + table.dartTypeName, _buffer); - if (table.columns.isNotEmpty) { - _buffer - ..write('&&') - ..write(table.columns.map((c) { - final getter = c.dartGetterName; - - return 'other.$getter == $getter'; - }).join(' && ')); - } - - // finish overrides method and class declaration - _buffer.write(');\n}'); + // finish class declaration + _buffer.write('}'); } void _writeMappingConstructor() { @@ -224,7 +212,7 @@ class DataClassWriter { _buffer.write('@override\n int get hashCode => '); final fields = table.columns.map((c) => c.dartGetterName).toList(); - HashCodeWriter().writeHashCode(fields, _buffer); + const HashCodeWriter().writeHashCode(fields, _buffer); _buffer.write(';'); } diff --git a/moor_generator/lib/src/writer/utils/hash_code.dart b/moor_generator/lib/src/writer/utils/hash_code.dart index aa59643a..81b2a0c0 100644 --- a/moor_generator/lib/src/writer/utils/hash_code.dart +++ b/moor_generator/lib/src/writer/utils/hash_code.dart @@ -2,6 +2,8 @@ const _hashCombine = '\$mrjc'; const _hashFinish = '\$mrjf'; class HashCodeWriter { + const HashCodeWriter(); + /// Writes an expression to calculate a hash code of an object that consists /// of the [fields]. void writeHashCode(List fields, StringBuffer into) { diff --git a/moor_generator/lib/src/writer/utils/override_equals.dart b/moor_generator/lib/src/writer/utils/override_equals.dart new file mode 100644 index 00000000..7c161252 --- /dev/null +++ b/moor_generator/lib/src/writer/utils/override_equals.dart @@ -0,0 +1,18 @@ +/// Writes a operator == override for a class consisting of the [fields] into +/// the buffer provided by [into]. +void overrideEquals( + Iterable fields, String className, StringBuffer into) { + into + ..write('@override\nbool operator ==(other) => ') + ..write('identical(this, other) || (other is $className'); + + if (fields.isNotEmpty) { + into + ..write(' && ') + ..write(fields.map((field) { + return 'other.$field == this.$field'; + }).join(' && ')); + } + + into.write(');\n'); +} diff --git a/moor_generator/test/writer/utils/hash_code_test.dart b/moor_generator/test/writer/utils/hash_code_test.dart index 008d0269..3431933c 100644 --- a/moor_generator/test/writer/utils/hash_code_test.dart +++ b/moor_generator/test/writer/utils/hash_code_test.dart @@ -4,19 +4,19 @@ import 'package:test_api/test_api.dart'; void main() { test('hash code for no fields', () { final buffer = StringBuffer(); - HashCodeWriter().writeHashCode([], buffer); + const HashCodeWriter().writeHashCode([], buffer); expect(buffer.toString(), r'identityHashCode(this)'); }); test('hash code for a single field', () { final buffer = StringBuffer(); - HashCodeWriter().writeHashCode(['a'], buffer); + const HashCodeWriter().writeHashCode(['a'], buffer); expect(buffer.toString(), r'$mrjf(a.hashCode)'); }); test('hash code for multiple fields', () { final buffer = StringBuffer(); - HashCodeWriter().writeHashCode(['a', 'b', 'c'], buffer); + const HashCodeWriter().writeHashCode(['a', 'b', 'c'], buffer); expect(buffer.toString(), r'$mrjf($mrjc(a.hashCode, $mrjc(b.hashCode, c.hashCode)))'); }); diff --git a/moor_generator/test/writer/utils/override_equals_test.dart b/moor_generator/test/writer/utils/override_equals_test.dart new file mode 100644 index 00000000..a4f3c204 --- /dev/null +++ b/moor_generator/test/writer/utils/override_equals_test.dart @@ -0,0 +1,25 @@ +import 'package:moor_generator/src/writer/utils/override_equals.dart'; +import 'package:test/test.dart'; + +void main() { + test('overrides equals on class without fields', () { + final buffer = StringBuffer(); + overrideEquals([], 'Foo', buffer); + + expect( + buffer.toString(), + '@override\nbool operator ==(other) => ' + 'identical(this, other) || (other is Foo);\n'); + }); + + test('overrides equals on class with fields', () { + final buffer = StringBuffer(); + overrideEquals(['a', 'b', 'c'], 'Foo', buffer); + + expect( + buffer.toString(), + '@override\nbool operator ==(other) => ' + 'identical(this, other) || (other is Foo && ' + 'other.a == this.a && other.b == this.b && other.c == this.c);\n'); + }); +} From 7d962a1f01ae336fe5f63ac6b866054137ac289b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 20 Sep 2019 19:36:09 +0200 Subject: [PATCH 111/117] Fix typo in new override flag --- docs/content/en/docs/Advanced Features/builder_options.md | 2 +- moor/build.yaml | 2 +- moor_generator/lib/src/backends/build/options.dart | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/en/docs/Advanced Features/builder_options.md b/docs/content/en/docs/Advanced Features/builder_options.md index 965b7146..4b27e858 100644 --- a/docs/content/en/docs/Advanced Features/builder_options.md +++ b/docs/content/en/docs/Advanced Features/builder_options.md @@ -26,6 +26,6 @@ At the moment, moor supports these options: * `write_from_json_string_constructor`: boolean. Adds a `.fromJsonString` factory constructor to generated data classes. By default, we only write a `.fromJson` constructor that takes a `Map`. -* `overrride_hash_and_equals_in_result_sets`: boolean. When moor generates another class +* `override_hash_and_equals_in_result_sets`: boolean. When moor generates another class to hold the result of generated select queries, this flag controls whether moor should override `operator ==` and `hashCode` in those classes. \ No newline at end of file diff --git a/moor/build.yaml b/moor/build.yaml index ac7bc057..5cbc6f98 100644 --- a/moor/build.yaml +++ b/moor/build.yaml @@ -3,4 +3,4 @@ targets: builders: moor_generator: options: - overrride_hash_and_equals_in_result_sets: true + override_hash_and_equals_in_result_sets: true diff --git a/moor_generator/lib/src/backends/build/options.dart b/moor_generator/lib/src/backends/build/options.dart index 3bac7b11..1cd2e630 100644 --- a/moor_generator/lib/src/backends/build/options.dart +++ b/moor_generator/lib/src/backends/build/options.dart @@ -12,7 +12,7 @@ class MoorOptions { config['write_from_json_string_constructor'] as bool ?? false; final overrideInResultSets = - config['overrride_hash_and_equals_in_result_sets'] as bool ?? false; + config['override_hash_and_equals_in_result_sets'] as bool ?? false; return MoorOptions(writeFromString, overrideInResultSets); } From 87c50de1e1ad53c371b473c3108c8020c9d6f1c1 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 20 Sep 2019 20:23:35 +0200 Subject: [PATCH 112/117] Add LazyDatabase wrapper to create a database async --- moor/CHANGELOG.md | 2 + moor/lib/moor.dart | 1 + moor/lib/src/utils/lazy_database.dart | 66 +++++++++++++++++++++++++ moor/test/data/utils/mocks.dart | 5 ++ moor/test/utils/lazy_database_test.dart | 48 ++++++++++++++++++ 5 files changed, 122 insertions(+) create mode 100644 moor/lib/src/utils/lazy_database.dart create mode 100644 moor/test/utils/lazy_database_test.dart diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 1c2db95e..57ab435e 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -54,6 +54,8 @@ TODO: Describe ffi port ### Minor changes - a `Constant` can now be written to SQL, it used to throw before. This is useful if you need default values for strings columns. +- new `LazyDatabase` when you want to construct a database asynchronously (for instance, if + you first need to find a file before you can open a database). ### Breaking changes - __THIS LIKELY AFFECTS YOUR APP:__ Removed the `transaction` parameter for callbacks diff --git a/moor/lib/moor.dart b/moor/lib/moor.dart index 6621ad3d..38440205 100644 --- a/moor/lib/moor.dart +++ b/moor/lib/moor.dart @@ -35,3 +35,4 @@ export 'package:moor/src/runtime/migration.dart'; export 'package:moor/src/runtime/exceptions.dart'; export 'package:moor/src/utils/expand_variables.dart'; export 'package:moor/src/utils/hash.dart'; +export 'package:moor/src/utils/lazy_database.dart'; diff --git a/moor/lib/src/utils/lazy_database.dart b/moor/lib/src/utils/lazy_database.dart new file mode 100644 index 00000000..16aa96ae --- /dev/null +++ b/moor/lib/src/utils/lazy_database.dart @@ -0,0 +1,66 @@ +import 'dart:async'; + +import 'package:moor/backends.dart'; + +/// Signature of a function that opens a database connection when instructed to. +typedef DatabaseOpener = FutureOr Function(); + +/// A special database executor that delegates work to another [QueryExecutor]. +/// The other executor is lazily opened by a [DatabaseOpener]. +class LazyDatabase extends QueryExecutor { + QueryExecutor _delegate; + Completer _openDelegate; + + /// The function that will open the database when this [LazyDatabase] gets + /// opened for the first time. + final DatabaseOpener opener; + + LazyDatabase(this.opener); + + Future _awaitOpened() { + if (_delegate != null) { + return Future.value(); + } else if (_openDelegate != null) { + return _openDelegate.future; + } else { + _openDelegate = Completer(); + Future.value(opener()).then((database) { + _delegate = database; + _openDelegate.complete(); + }); + return _openDelegate.future; + } + } + + @override + TransactionExecutor beginTransaction() => _delegate.beginTransaction(); + + @override + Future ensureOpen() { + return _awaitOpened().then((_) => _delegate.ensureOpen()); + } + + @override + Future runBatched(List statements) => + _delegate.runBatched(statements); + + @override + Future runCustom(String statement, [List args]) => + _delegate.runCustom(statement, args); + + @override + Future runDelete(String statement, List args) => + _delegate.runDelete(statement, args); + + @override + Future runInsert(String statement, List args) => + _delegate.runInsert(statement, args); + + @override + Future>> runSelect(String statement, List args) => + _delegate.runSelect(statement, args); + + @override + Future runUpdate(String statement, List args) => + _delegate.runUpdate(statement, args); +} diff --git a/moor/test/data/utils/mocks.dart b/moor/test/data/utils/mocks.dart index 23566853..2196cb28 100644 --- a/moor/test/data/utils/mocks.dart +++ b/moor/test/data/utils/mocks.dart @@ -34,6 +34,11 @@ class MockExecutor extends Mock implements QueryExecutor { return transactions; }); + when(ensureOpen()).thenAnswer((i) { + _opened = true; + return Future.value(true); + }); + when(doWhenOpened(any)).thenAnswer((i) { _opened = true; final action = i.positionalArguments.single as _EnsureOpenAction; diff --git a/moor/test/utils/lazy_database_test.dart b/moor/test/utils/lazy_database_test.dart new file mode 100644 index 00000000..326d2012 --- /dev/null +++ b/moor/test/utils/lazy_database_test.dart @@ -0,0 +1,48 @@ +import 'package:moor/moor.dart'; +import 'package:pedantic/pedantic.dart'; +import 'package:test_api/test_api.dart'; + +import '../data/utils/mocks.dart'; + +void main() { + test('lazy database delegates work', () async { + final inner = MockExecutor(); + final lazy = LazyDatabase(() => inner); + + await lazy.ensureOpen(); + clearInteractions(inner); + + lazy.beginTransaction(); + await lazy.runBatched(null); + await lazy.runCustom('custom_stmt'); + await lazy.runDelete('delete_stmt', [1]); + await lazy.runInsert('insert_stmt', [2]); + await lazy.runSelect('select_stmt', [3]); + await lazy.runUpdate('update_stmt', [4]); + + verifyInOrder([ + inner.runBatched(null), + inner.runCustom('custom_stmt'), + inner.runDelete('delete_stmt', [1]), + inner.runInsert('insert_stmt', [2]), + inner.runSelect('select_stmt', [3]), + inner.runUpdate('update_stmt', [4]), + ]); + }); + + test('database is only opened once', () async { + final inner = MockExecutor(); + var openCount = 0; + final lazy = LazyDatabase(() { + openCount++; + return inner; + }); + + for (var i = 0; i < 10; i++) { + unawaited(lazy.ensureOpen()); + } + + await pumpEventQueue(); + expect(openCount, 1); + }); +} From b62ed05a87957df133afc7be2a102a26a9859855 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Fri, 20 Sep 2019 20:46:54 +0200 Subject: [PATCH 113/117] Make the lazy executor pass on the GeneratedDatabase --- moor/lib/src/utils/lazy_database.dart | 8 ++++++++ moor/test/utils/lazy_database_test.dart | 11 +++++++++++ 2 files changed, 19 insertions(+) diff --git a/moor/lib/src/utils/lazy_database.dart b/moor/lib/src/utils/lazy_database.dart index 16aa96ae..185d1c93 100644 --- a/moor/lib/src/utils/lazy_database.dart +++ b/moor/lib/src/utils/lazy_database.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'package:moor/backends.dart'; +import 'package:moor/moor.dart'; /// Signature of a function that opens a database connection when instructed to. typedef DatabaseOpener = FutureOr Function(); @@ -17,6 +18,12 @@ class LazyDatabase extends QueryExecutor { LazyDatabase(this.opener); + @override + set databaseInfo(GeneratedDatabase db) { + super.databaseInfo = db; + _delegate?.databaseInfo = db; + } + Future _awaitOpened() { if (_delegate != null) { return Future.value(); @@ -26,6 +33,7 @@ class LazyDatabase extends QueryExecutor { _openDelegate = Completer(); Future.value(opener()).then((database) { _delegate = database; + _delegate.databaseInfo = databaseInfo; _openDelegate.complete(); }); return _openDelegate.future; diff --git a/moor/test/utils/lazy_database_test.dart b/moor/test/utils/lazy_database_test.dart index 326d2012..2ce480f1 100644 --- a/moor/test/utils/lazy_database_test.dart +++ b/moor/test/utils/lazy_database_test.dart @@ -2,6 +2,7 @@ import 'package:moor/moor.dart'; import 'package:pedantic/pedantic.dart'; import 'package:test_api/test_api.dart'; +import '../data/tables/todos.dart'; import '../data/utils/mocks.dart'; void main() { @@ -45,4 +46,14 @@ void main() { await pumpEventQueue(); expect(openCount, 1); }); + + test('sets generated database property', () async { + final inner = MockExecutor(); + final db = TodoDb(LazyDatabase(() => inner)); + + // run a statement to make sure the database has been opened + await db.customSelectQuery('custom_select').get(); + + verify(inner.databaseInfo = db); + }); } From 22dee726803061923f09033512390b8ad73ecf03 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sat, 21 Sep 2019 13:20:19 +0200 Subject: [PATCH 114/117] Bump sqflite dependency, integration test for array args --- docs/content/en/docs/Examples/_index.md | 3 +- .../integration_tests/flutter_db/pubspec.lock | 51 ++++++-------- .../tests/analysis_options.yaml | 3 - .../tests/lib/database/database.dart | 1 + .../tests/lib/database/database.g.dart | 35 +++++++--- .../tests/lib/database/test.moor | 3 - .../tests/lib/suite/crud_tests.dart | 9 +++ moor_flutter/pubspec.lock | 69 ++++++++++++++++--- moor_flutter/pubspec.yaml | 2 +- 9 files changed, 120 insertions(+), 56 deletions(-) delete mode 100644 extras/integration_tests/tests/analysis_options.yaml delete mode 100644 extras/integration_tests/tests/lib/database/test.moor diff --git a/docs/content/en/docs/Examples/_index.md b/docs/content/en/docs/Examples/_index.md index b2adf085..54651eea 100644 --- a/docs/content/en/docs/Examples/_index.md +++ b/docs/content/en/docs/Examples/_index.md @@ -7,6 +7,7 @@ description: Example apps using moor We have an [example in the repo](https://github.com/simolus3/moor/tree/master/moor_flutter/example), it's a simple todo list app, -written with moor. +written with moor. [Rody Davis](https://github.com/AppleEducate) has built a cleaner version of the example that works on all +Flutter platforms - including Web and Desktop! You can check it out [here](https://github.com/AppleEducate/moor_shared). The [HackerNews reader app](https://github.com/filiph/hn_app) from the [Boring Flutter Show](https://www.youtube.com/playlist?list=PLjxrf2q8roU3ahJVrSgAnPjzkpGmL9Czl) also uses moor to keep a list of favorite articles. \ No newline at end of file diff --git a/extras/integration_tests/flutter_db/pubspec.lock b/extras/integration_tests/flutter_db/pubspec.lock index 04a3d533..dd25145c 100644 --- a/extras/integration_tests/flutter_db/pubspec.lock +++ b/extras/integration_tests/flutter_db/pubspec.lock @@ -7,7 +7,7 @@ packages: name: analyzer url: "https://pub.dartlang.org" source: hosted - version: "0.36.4" + version: "0.38.3" args: dependency: transitive description: @@ -21,14 +21,14 @@ packages: name: async url: "https://pub.dartlang.org" source: hosted - version: "2.2.0" + version: "2.3.0" boolean_selector: dependency: transitive description: name: boolean_selector url: "https://pub.dartlang.org" source: hosted - version: "1.0.4" + version: "1.0.5" charcode: dependency: transitive description: @@ -56,7 +56,7 @@ packages: name: crypto url: "https://pub.dartlang.org" source: hosted - version: "2.0.6" + version: "2.1.3" csslib: dependency: transitive description: @@ -75,7 +75,7 @@ packages: name: front_end url: "https://pub.dartlang.org" source: hosted - version: "0.1.19" + version: "0.1.25" glob: dependency: transitive description: @@ -131,21 +131,14 @@ packages: name: json_annotation url: "https://pub.dartlang.org" source: hosted - version: "2.4.0" - json_rpc_2: - dependency: transitive - description: - name: json_rpc_2 - url: "https://pub.dartlang.org" - source: hosted - version: "2.1.0" + version: "3.0.0" kernel: dependency: transitive description: name: kernel url: "https://pub.dartlang.org" source: hosted - version: "0.3.19" + version: "0.3.25" matcher: dependency: transitive description: @@ -201,14 +194,14 @@ packages: name: node_preamble url: "https://pub.dartlang.org" source: hosted - version: "1.4.5" + version: "1.4.8" package_config: dependency: transitive description: name: package_config url: "https://pub.dartlang.org" source: hosted - version: "1.0.5" + version: "1.1.0" package_resolver: dependency: transitive description: @@ -222,14 +215,14 @@ packages: name: path url: "https://pub.dartlang.org" source: hosted - version: "1.6.2" + version: "1.6.4" pedantic: dependency: transitive description: name: pedantic url: "https://pub.dartlang.org" source: hosted - version: "1.7.0" + version: "1.8.0+1" pool: dependency: transitive description: @@ -304,7 +297,7 @@ packages: name: sqflite url: "https://pub.dartlang.org" source: hosted - version: "1.1.6+2" + version: "1.1.6+5" stack_trace: dependency: transitive description: @@ -325,7 +318,7 @@ packages: name: string_scanner url: "https://pub.dartlang.org" source: hosted - version: "1.0.4" + version: "1.0.5" synchronized: dependency: transitive description: @@ -346,21 +339,21 @@ packages: name: test url: "https://pub.dartlang.org" source: hosted - version: "1.6.3" + version: "1.6.10" test_api: dependency: transitive description: name: test_api url: "https://pub.dartlang.org" source: hosted - version: "0.2.5" + version: "0.2.7" test_core: dependency: transitive description: name: test_core url: "https://pub.dartlang.org" source: hosted - version: "0.2.5" + version: "0.2.9+1" tests: dependency: "direct main" description: @@ -382,13 +375,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.0.8" - vm_service_client: + vm_service: dependency: transitive description: - name: vm_service_client + name: vm_service url: "https://pub.dartlang.org" source: hosted - version: "0.2.6+2" + version: "2.0.0" watcher: dependency: transitive description: @@ -402,14 +395,14 @@ packages: name: web_socket_channel url: "https://pub.dartlang.org" source: hosted - version: "1.0.13" + version: "1.0.15" yaml: dependency: transitive description: name: yaml url: "https://pub.dartlang.org" source: hosted - version: "2.1.16" + version: "2.2.0" sdks: - dart: ">=2.5.0-dev <2.6.0" + dart: ">=2.5.0-dev <=2.6.0-dev.1.0.flutter-7c1821c4aa" flutter: ">=1.2.1 <2.0.0" diff --git a/extras/integration_tests/tests/analysis_options.yaml b/extras/integration_tests/tests/analysis_options.yaml deleted file mode 100644 index e649ddc0..00000000 --- a/extras/integration_tests/tests/analysis_options.yaml +++ /dev/null @@ -1,3 +0,0 @@ -analyzer: - plugins: - - moor \ No newline at end of file diff --git a/extras/integration_tests/tests/lib/database/database.dart b/extras/integration_tests/tests/lib/database/database.dart index 089890fe..5e10265c 100644 --- a/extras/integration_tests/tests/lib/database/database.dart +++ b/extras/integration_tests/tests/lib/database/database.dart @@ -75,6 +75,7 @@ class PreferenceConverter extends TypeConverter { WHERE (f.first_user = :user OR f.second_user = :user)''', 'userCount': 'SELECT COUNT(id) FROM users', 'settingsFor': 'SELECT preferences FROM users WHERE id = :user', + 'usersById': 'SELECT * FROM users WHERE id IN ?', }, ) class Database extends _$Database { diff --git a/extras/integration_tests/tests/lib/database/database.g.dart b/extras/integration_tests/tests/lib/database/database.g.dart index 8e5c09ed..c32c4097 100644 --- a/extras/integration_tests/tests/lib/database/database.g.dart +++ b/extras/integration_tests/tests/lib/database/database.g.dart @@ -21,7 +21,7 @@ Map _$PreferencesToJson(Preferences instance) => // MoorGenerator // ************************************************************************** -// ignore_for_file: unnecessary_brace_in_string_interps +// ignore_for_file: unnecessary_brace_in_string_interps, unnecessary_this class User extends DataClass implements Insertable { final int id; final String name; @@ -127,11 +127,11 @@ class User extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is User && - other.id == id && - other.name == name && - other.birthDate == birthDate && - other.profilePicture == profilePicture && - other.preferences == preferences); + other.id == this.id && + other.name == this.name && + other.birthDate == this.birthDate && + other.profilePicture == this.profilePicture && + other.preferences == this.preferences); } class UsersCompanion extends UpdateCompanion { @@ -397,9 +397,9 @@ class Friendship extends DataClass implements Insertable { bool operator ==(other) => identical(this, other) || (other is Friendship && - other.firstUser == firstUser && - other.secondUser == secondUser && - other.reallyGoodFriends == reallyGoodFriends); + other.firstUser == this.firstUser && + other.secondUser == this.secondUser && + other.reallyGoodFriends == this.reallyGoodFriends); } class FriendshipsCompanion extends UpdateCompanion { @@ -630,6 +630,23 @@ abstract class _$Database extends GeneratedDatabase { return settingsForQuery(user).watch(); } + Selectable usersByIdQuery(List var1) { + var $arrayStartIndex = 1; + final expandedvar1 = $expandVar($arrayStartIndex, var1.length); + $arrayStartIndex += var1.length; + return customSelectQuery('SELECT * FROM users WHERE id IN ($expandedvar1)', + variables: [for (var $ in var1) Variable.withInt($)], + readsFrom: {users}).map(_rowToUser); + } + + Future> usersById(List var1) { + return usersByIdQuery(var1).get(); + } + + Stream> watchUsersById(List var1) { + return usersByIdQuery(var1).watch(); + } + @override List get allTables => [users, friendships]; } diff --git a/extras/integration_tests/tests/lib/database/test.moor b/extras/integration_tests/tests/lib/database/test.moor deleted file mode 100644 index e2d524e3..00000000 --- a/extras/integration_tests/tests/lib/database/test.moor +++ /dev/null @@ -1,3 +0,0 @@ -CREATE TABLE test ( - id INT NOT NULL PRIMARY AUTOINCREMENT -) \ No newline at end of file diff --git a/extras/integration_tests/tests/lib/suite/crud_tests.dart b/extras/integration_tests/tests/lib/suite/crud_tests.dart index 1495bf41..4ab96378 100644 --- a/extras/integration_tests/tests/lib/suite/crud_tests.dart +++ b/extras/integration_tests/tests/lib/suite/crud_tests.dart @@ -23,4 +23,13 @@ void crudTests(TestExecutor executor) { await db.makeFriends(a, b); await expectation; }); + + test('IN ? expressions can be expanded', () async { + // regression test for https://github.com/simolus3/moor/issues/156 + final db = Database(executor.createExecutor()); + + final result = await db.usersById([1, 2, 3]); + + expect(result.map((u) => u.name), ['Dash', 'Duke', 'Go Gopher']); + }); } diff --git a/moor_flutter/pubspec.lock b/moor_flutter/pubspec.lock index 5837c258..ea2eba3e 100644 --- a/moor_flutter/pubspec.lock +++ b/moor_flutter/pubspec.lock @@ -1,20 +1,34 @@ # Generated by pub # See https://dart.dev/tools/pub/glossary#lockfile packages: + archive: + dependency: transitive + description: + name: archive + url: "https://pub.dartlang.org" + source: hosted + version: "2.0.10" + args: + dependency: transitive + description: + name: args + url: "https://pub.dartlang.org" + source: hosted + version: "1.5.2" async: dependency: transitive description: name: async url: "https://pub.dartlang.org" source: hosted - version: "2.2.0" + version: "2.3.0" boolean_selector: dependency: transitive description: name: boolean_selector url: "https://pub.dartlang.org" source: hosted - version: "1.0.4" + version: "1.0.5" charcode: dependency: transitive description: @@ -29,6 +43,20 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.14.11" + convert: + dependency: transitive + description: + name: convert + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.1" + crypto: + dependency: transitive + description: + name: crypto + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.3" flutter: dependency: "direct main" description: flutter @@ -39,6 +67,13 @@ packages: description: flutter source: sdk version: "0.0.0" + image: + dependency: transitive + description: + name: image + url: "https://pub.dartlang.org" + source: hosted + version: "2.1.4" matcher: dependency: transitive description: @@ -52,35 +87,42 @@ packages: name: meta url: "https://pub.dartlang.org" source: hosted - version: "1.1.6" + version: "1.1.7" moor: dependency: "direct main" description: path: "../moor" relative: true source: path - version: "1.7.1" + version: "1.7.2" path: dependency: "direct main" description: name: path url: "https://pub.dartlang.org" source: hosted - version: "1.6.2" + version: "1.6.4" pedantic: dependency: transitive description: name: pedantic url: "https://pub.dartlang.org" source: hosted - version: "1.7.0" + version: "1.8.0+1" + petitparser: + dependency: transitive + description: + name: petitparser + url: "https://pub.dartlang.org" + source: hosted + version: "2.4.0" quiver: dependency: transitive description: name: quiver url: "https://pub.dartlang.org" source: hosted - version: "2.0.3" + version: "2.0.5" sky_engine: dependency: transitive description: flutter @@ -99,7 +141,7 @@ packages: name: sqflite url: "https://pub.dartlang.org" source: hosted - version: "1.1.6+4" + version: "1.1.6+5" stack_trace: dependency: transitive description: @@ -120,7 +162,7 @@ packages: name: string_scanner url: "https://pub.dartlang.org" source: hosted - version: "1.0.4" + version: "1.0.5" synchronized: dependency: transitive description: @@ -156,6 +198,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.0.8" + xml: + dependency: transitive + description: + name: xml + url: "https://pub.dartlang.org" + source: hosted + version: "3.5.0" sdks: - dart: ">=2.2.2 <3.0.0" + dart: ">=2.4.0 <3.0.0" flutter: ">=1.2.1 <2.0.0" diff --git a/moor_flutter/pubspec.yaml b/moor_flutter/pubspec.yaml index 9b55aba3..762af402 100644 --- a/moor_flutter/pubspec.yaml +++ b/moor_flutter/pubspec.yaml @@ -13,7 +13,7 @@ environment: dependencies: moor: ^1.7.0 - sqflite: ^1.1.0 + sqflite: ^1.1.6+5 meta: ^1.0.0 path: ^1.0.0 flutter: From d6913af380cad5b282180071119c28e5032793af Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 22 Sep 2019 11:10:32 +0200 Subject: [PATCH 115/117] FFI: Ability to override loading behavior, async api --- moor_ffi/lib/database.dart | 6 + moor_ffi/lib/moor_ffi.dart | 2 +- moor_ffi/lib/src/api/database.dart | 165 +++--------------- moor_ffi/lib/src/api/result.dart | 4 +- moor_ffi/lib/src/bindings/bindings.dart | 2 +- moor_ffi/lib/src/impl/database.dart | 156 +++++++++++++++++ moor_ffi/lib/src/{api => impl}/errors.dart | 0 .../src/{api => impl}/prepared_statement.dart | 8 +- moor_ffi/lib/src/load_library.dart | 79 ++++++++- moor_ffi/lib/src/vm_database.dart | 51 +++--- 10 files changed, 294 insertions(+), 179 deletions(-) create mode 100644 moor_ffi/lib/database.dart create mode 100644 moor_ffi/lib/src/impl/database.dart rename moor_ffi/lib/src/{api => impl}/errors.dart (100%) rename moor_ffi/lib/src/{api => impl}/prepared_statement.dart (95%) diff --git a/moor_ffi/lib/database.dart b/moor_ffi/lib/database.dart new file mode 100644 index 00000000..9b6157eb --- /dev/null +++ b/moor_ffi/lib/database.dart @@ -0,0 +1,6 @@ +/// Exports the raw [] +library database; + +export 'src/api/database.dart'; +export 'src/api/result.dart'; +export 'src/impl/database.dart' show SqliteException, Database; diff --git a/moor_ffi/lib/moor_ffi.dart b/moor_ffi/lib/moor_ffi.dart index 632b3f81..7954e761 100644 --- a/moor_ffi/lib/moor_ffi.dart +++ b/moor_ffi/lib/moor_ffi.dart @@ -3,7 +3,7 @@ import 'dart:io'; import 'package:moor/backends.dart'; import 'package:moor/moor.dart'; -import 'package:moor_ffi/src/api/database.dart'; +import 'package:moor_ffi/database.dart'; part 'src/vm_database.dart'; part 'src/load_library.dart'; diff --git a/moor_ffi/lib/src/api/database.dart b/moor_ffi/lib/src/api/database.dart index 96633e2a..bd54c1a1 100644 --- a/moor_ffi/lib/src/api/database.dart +++ b/moor_ffi/lib/src/api/database.dart @@ -1,161 +1,44 @@ -import 'dart:collection'; -import 'dart:ffi'; -import 'dart:io'; -import 'dart:typed_data'; +import 'dart:async'; -import 'package:collection/collection.dart'; -import 'package:moor_ffi/src/bindings/constants.dart'; -import 'package:moor_ffi/src/bindings/types.dart' as types; -import 'package:moor_ffi/src/bindings/bindings.dart'; -import 'package:moor_ffi/src/ffi/blob.dart'; -import 'package:moor_ffi/src/ffi/utils.dart'; - -part 'errors.dart'; -part 'prepared_statement.dart'; -part 'result.dart'; - -const _openingFlags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE; - -class Database { - final Pointer _db; - final List _preparedStmt = []; - bool _isClosed = false; - - Database._(this._db); - - /// Opens the [file] as a sqlite3 database. The file will be created if it - /// doesn't exist. - factory Database.openFile(File file) => Database.open(file.absolute.path); - - /// Opens an in-memory sqlite3 database. - factory Database.memory() => Database.open(':memory:'); - - /// Opens an sqlite3 database from a filename. - factory Database.open(String fileName) { - final dbOut = Pointer>.allocate(); - final pathC = CBlob.allocateString(fileName); - - final resultCode = - bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, nullptr.cast()); - final dbPointer = dbOut.load>(); - - dbOut.free(); - pathC.free(); - - if (resultCode == Errors.SQLITE_OK) { - return Database._(dbPointer); - } else { - throw SqliteException._fromErrorCode(dbPointer, resultCode); - } - } - - void _ensureOpen() { - if (_isClosed) { - throw Exception('This database has already been closed'); - } - } +import 'package:moor_ffi/database.dart'; +/// A opened sqlite database. +abstract class BaseDatabase { /// Closes this database connection and releases the resources it uses. If /// an error occurs while closing the database, an exception will be thrown. /// The allocated memory will be freed either way. - void close() { - final code = bindings.sqlite3_close_v2(_db); - SqliteException exception; - if (code != Errors.SQLITE_OK) { - exception = SqliteException._fromErrorCode(_db, code); - } - _isClosed = true; - - for (var stmt in _preparedStmt) { - stmt.close(); - } - _db.free(); - - if (exception != null) { - throw exception; - } - } - - void _handleStmtFinalized(PreparedStatement stmt) { - if (!_isClosed) { - _preparedStmt.remove(stmt); - } - } + FutureOr close(); /// Executes the [sql] statement and ignores the result. Will throw if an /// error occurs while executing. - void execute(String sql) { - _ensureOpen(); - final sqlPtr = CBlob.allocateString(sql); - final errorOut = Pointer>.allocate(); - - final result = - bindings.sqlite3_exec(_db, sqlPtr, nullptr, nullptr, errorOut); - - sqlPtr.free(); - - final errorPtr = errorOut.load>(); - errorOut.free(); - - String errorMsg; - if (!isNullPointer(errorPtr)) { - errorMsg = errorPtr.load().readString(); - // the message was allocated from sqlite, we need to free it - bindings.sqlite3_free(errorPtr.cast()); - } - - if (result != Errors.SQLITE_OK) { - throw SqliteException(errorMsg); - } - } + FutureOr execute(String sql); /// Prepares the [sql] statement. - PreparedStatement prepare(String sql) { - _ensureOpen(); - - final stmtOut = Pointer>.allocate(); - final sqlPtr = CBlob.allocateString(sql); - - final resultCode = - bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, nullptr.cast()); - sqlPtr.free(); - - final stmt = stmtOut.load>(); - stmtOut.free(); - - if (resultCode != Errors.SQLITE_OK) { - // we don't need to worry about freeing the statement. If preparing the - // statement was unsuccessful, stmtOut.load() will be null - throw SqliteException._fromErrorCode(_db, resultCode); - } - - return PreparedStatement._(stmt, this); - } + FutureOr prepare(String sql); /// Get the application defined version of this database. - int get userVersion { - final stmt = prepare('PRAGMA user_version'); - final result = stmt.select(); - stmt.close(); - - return result.first.columnAt(0) as int; - } + FutureOr userVersion(); /// Update the application defined version of this database. - set userVersion(int version) { - execute('PRAGMA user_version = $version'); - } + FutureOr setUserVersion(int version); /// Returns the amount of rows affected by the last INSERT, UPDATE or DELETE /// statement. - int get updatedRows { - _ensureOpen(); - return bindings.sqlite3_changes(_db); - } + FutureOr getUpdatedRows(); /// Returns the row-id of the last inserted row. - int get lastInsertId { - _ensureOpen(); - return bindings.sqlite3_last_insert_rowid(_db); - } + FutureOr getLastInsertId(); +} + +/// A prepared statement that can be executed multiple times. +abstract class BasePreparedStatement { + /// Executes this prepared statement as a select statement. The returned rows + /// will be returned. + FutureOr select([List args]); + + /// Executes this prepared statement. + FutureOr execute([List params]); + + /// Closes this prepared statement and releases its resources. + FutureOr close(); } diff --git a/moor_ffi/lib/src/api/result.dart b/moor_ffi/lib/src/api/result.dart index 3204124a..9c90cd2d 100644 --- a/moor_ffi/lib/src/api/result.dart +++ b/moor_ffi/lib/src/api/result.dart @@ -1,4 +1,6 @@ -part of 'database.dart'; +import 'dart:collection'; + +import 'package:collection/collection.dart'; /// Stores the result of a select statement. class Result extends Iterable { diff --git a/moor_ffi/lib/src/bindings/bindings.dart b/moor_ffi/lib/src/bindings/bindings.dart index 7008699d..4b397358 100644 --- a/moor_ffi/lib/src/bindings/bindings.dart +++ b/moor_ffi/lib/src/bindings/bindings.dart @@ -94,7 +94,7 @@ class _SQLiteBindings { int Function(Pointer statement, int columnIndex) sqlite3_bind_null; _SQLiteBindings() { - sqlite = moorSqliteOpener(); + sqlite = open.openSqlite(); sqlite3_bind_double = sqlite .lookup>( diff --git a/moor_ffi/lib/src/impl/database.dart b/moor_ffi/lib/src/impl/database.dart new file mode 100644 index 00000000..1878013b --- /dev/null +++ b/moor_ffi/lib/src/impl/database.dart @@ -0,0 +1,156 @@ +import 'dart:ffi'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:moor_ffi/database.dart'; +import 'package:moor_ffi/src/api/result.dart'; +import 'package:moor_ffi/src/bindings/constants.dart'; +import 'package:moor_ffi/src/bindings/types.dart' as types; +import 'package:moor_ffi/src/bindings/bindings.dart'; +import 'package:moor_ffi/src/ffi/blob.dart'; +import 'package:moor_ffi/src/ffi/utils.dart'; + +part 'errors.dart'; +part 'prepared_statement.dart'; + +const _openingFlags = Flags.SQLITE_OPEN_READWRITE | Flags.SQLITE_OPEN_CREATE; + +class Database implements BaseDatabase { + final Pointer _db; + final List _preparedStmt = []; + bool _isClosed = false; + + Database._(this._db); + + /// Opens the [file] as a sqlite3 database. The file will be created if it + /// doesn't exist. + factory Database.openFile(File file) => Database.open(file.absolute.path); + + /// Opens an in-memory sqlite3 database. + factory Database.memory() => Database.open(':memory:'); + + /// Opens an sqlite3 database from a filename. + factory Database.open(String fileName) { + final dbOut = Pointer>.allocate(); + final pathC = CBlob.allocateString(fileName); + + final resultCode = + bindings.sqlite3_open_v2(pathC, dbOut, _openingFlags, nullptr.cast()); + final dbPointer = dbOut.load>(); + + dbOut.free(); + pathC.free(); + + if (resultCode == Errors.SQLITE_OK) { + return Database._(dbPointer); + } else { + throw SqliteException._fromErrorCode(dbPointer, resultCode); + } + } + + void _ensureOpen() { + if (_isClosed) { + throw Exception('This database has already been closed'); + } + } + + @override + void close() { + final code = bindings.sqlite3_close_v2(_db); + SqliteException exception; + if (code != Errors.SQLITE_OK) { + exception = SqliteException._fromErrorCode(_db, code); + } + _isClosed = true; + + for (var stmt in _preparedStmt) { + stmt.close(); + } + _db.free(); + + if (exception != null) { + throw exception; + } + } + + void _handleStmtFinalized(PreparedStatement stmt) { + if (!_isClosed) { + _preparedStmt.remove(stmt); + } + } + + @override + void execute(String sql) { + _ensureOpen(); + final sqlPtr = CBlob.allocateString(sql); + final errorOut = Pointer>.allocate(); + + final result = + bindings.sqlite3_exec(_db, sqlPtr, nullptr, nullptr, errorOut); + + sqlPtr.free(); + + final errorPtr = errorOut.load>(); + errorOut.free(); + + String errorMsg; + if (!isNullPointer(errorPtr)) { + errorMsg = errorPtr.load().readString(); + // the message was allocated from sqlite, we need to free it + bindings.sqlite3_free(errorPtr.cast()); + } + + if (result != Errors.SQLITE_OK) { + throw SqliteException(errorMsg); + } + } + + @override + PreparedStatement prepare(String sql) { + _ensureOpen(); + + final stmtOut = Pointer>.allocate(); + final sqlPtr = CBlob.allocateString(sql); + + final resultCode = + bindings.sqlite3_prepare_v2(_db, sqlPtr, -1, stmtOut, nullptr.cast()); + sqlPtr.free(); + + final stmt = stmtOut.load>(); + stmtOut.free(); + + if (resultCode != Errors.SQLITE_OK) { + // we don't need to worry about freeing the statement. If preparing the + // statement was unsuccessful, stmtOut.load() will be null + throw SqliteException._fromErrorCode(_db, resultCode); + } + + return PreparedStatement._(stmt, this); + } + + @override + int userVersion() { + final stmt = prepare('PRAGMA user_version'); + final result = stmt.select(); + stmt.close(); + + return result.first.columnAt(0) as int; + } + + @override + void setUserVersion(int version) { + execute('PRAGMA user_version = $version'); + } + + @override + int getUpdatedRows() { + _ensureOpen(); + return bindings.sqlite3_changes(_db); + } + + @override + int getLastInsertId() { + _ensureOpen(); + return bindings.sqlite3_last_insert_rowid(_db); + } +} diff --git a/moor_ffi/lib/src/api/errors.dart b/moor_ffi/lib/src/impl/errors.dart similarity index 100% rename from moor_ffi/lib/src/api/errors.dart rename to moor_ffi/lib/src/impl/errors.dart diff --git a/moor_ffi/lib/src/api/prepared_statement.dart b/moor_ffi/lib/src/impl/prepared_statement.dart similarity index 95% rename from moor_ffi/lib/src/api/prepared_statement.dart rename to moor_ffi/lib/src/impl/prepared_statement.dart index 5b06d476..01ff6ffa 100644 --- a/moor_ffi/lib/src/api/prepared_statement.dart +++ b/moor_ffi/lib/src/impl/prepared_statement.dart @@ -1,6 +1,6 @@ part of 'database.dart'; -class PreparedStatement { +class PreparedStatement implements BasePreparedStatement { final Pointer _stmt; final Database _db; bool _closed = false; @@ -10,6 +10,7 @@ class PreparedStatement { PreparedStatement._(this._stmt, this._db); + @override void close() { if (!_closed) { _reset(); @@ -25,8 +26,7 @@ class PreparedStatement { } } - /// Executes this prepared statement as a select statement. The returned rows - /// will be returned. + @override Result select([List params]) { _ensureNotFinalized(); _reset(); @@ -75,7 +75,7 @@ class PreparedStatement { } } - /// Executes this prepared statement. + @override void execute([List params]) { _ensureNotFinalized(); _reset(); diff --git a/moor_ffi/lib/src/load_library.dart b/moor_ffi/lib/src/load_library.dart index c3bdaea4..db3108e0 100644 --- a/moor_ffi/lib/src/load_library.dart +++ b/moor_ffi/lib/src/load_library.dart @@ -4,11 +4,19 @@ part of 'package:moor_ffi/moor_ffi.dart'; /// use. typedef OpenLibrary = DynamicLibrary Function(); -/// The [OpenLibrary] function that will be used for the first time the native -/// library is requested. This can be overridden, but won't have an effect after -/// the library has been opened once (which happens when a `VmDatabase` is -/// instantiated). -OpenLibrary moorSqliteOpener = _defaultOpen; +enum OperatingSystem { + android, + linux, + iOS, + macOS, + windows, + fuchsia, +} + +/// The instance managing different approaches to load the [DynamicLibrary] for +/// sqlite when needed. See the documentation for [OpenDynamicLibrary] to learn +/// how the default opening behavior can be overridden. +final OpenDynamicLibrary open = OpenDynamicLibrary._(); DynamicLibrary _defaultOpen() { if (Platform.isLinux || Platform.isAndroid) { @@ -26,3 +34,64 @@ DynamicLibrary _defaultOpen() { throw UnsupportedError( 'moor_ffi does not support ${Platform.operatingSystem} yet'); } + +/// Manages functions that define how to load the [DynamicLibrary] for sqlite. +/// +/// The default behavior will use `DynamicLibrary.open('libsqlite3.so')` on +/// Linux and Android, `DynamicLibrary.open('libsqlite3.dylib')` on iOS and +/// macOS and `DynamicLibrary.open('sqlite3.dll')` on Windows. +/// +/// The default behavior can be overridden for a specific OS by using +/// [overrideFor]. To override the behavior on all platforms, use +/// [overrideForAll]. +class OpenDynamicLibrary { + final Map _overriddenPlatforms = {}; + OpenLibrary _overriddenForAll; + + OpenDynamicLibrary._(); + + /// Returns the current [OperatingSystem] as read from the [Platform] getters. + OperatingSystem get os { + if (Platform.isAndroid) return OperatingSystem.android; + if (Platform.isLinux) return OperatingSystem.linux; + if (Platform.isIOS) return OperatingSystem.iOS; + if (Platform.isMacOS) return OperatingSystem.macOS; + if (Platform.isWindows) return OperatingSystem.windows; + if (Platform.isFuchsia) return OperatingSystem.fuchsia; + return null; + } + + /// Opens the [DynamicLibrary] from which `moor_ffi` is going to + /// [DynamicLibrary.lookup] sqlite's methods that will be used. This method is + /// meant to be called by `moor_ffi` only. + DynamicLibrary openSqlite() { + if (_overriddenForAll != null) { + return _overriddenForAll(); + } + + final forPlatform = _overriddenPlatforms[os]; + if (forPlatform != null) { + return forPlatform(); + } + + return _defaultOpen(); + } + + /// Makes `moor_ffi` use the [open] function when running on the specified + /// [os]. This can be used to override the loading behavior on some platforms. + /// To override that behavior on all platforms, consider using + /// [overrideForAll]. + /// This method must be called before opening any database. + /// + /// When using the asynchronous API over isolates, [open] __must be__ a top- + /// level function or a static method. + void overrideFor(OperatingSystem os, OpenLibrary open) {} + + /// Makes `moor_ffi` use the [OpenLibrary] function for all Dart platforms. + /// If this method has been called, it takes precedence over [overrideFor]. + /// This method must be called before opening any database. + /// + /// When using the asynchronous API over isolates, [open] __must be__ a top- + /// level function or a static method. + void overrideForAll(OpenLibrary open) {} +} diff --git a/moor_ffi/lib/src/vm_database.dart b/moor_ffi/lib/src/vm_database.dart index c73a8a5e..4699c898 100644 --- a/moor_ffi/lib/src/vm_database.dart +++ b/moor_ffi/lib/src/vm_database.dart @@ -18,7 +18,7 @@ class VmDatabase extends DelegatedDatabase { } class _VmDelegate extends DatabaseDelegate { - Database _db; + BaseDatabase _db; final File file; @@ -45,12 +45,12 @@ class _VmDelegate extends DatabaseDelegate { } @override - Future runBatched(List statements) { + Future runBatched(List statements) async { for (var stmt in statements) { - final prepared = _db.prepare(stmt.sql); + final prepared = await _db.prepare(stmt.sql); for (var boundVars in stmt.variables) { - prepared.execute(boundVars); + await prepared.execute(boundVars); } prepared.close(); @@ -59,55 +59,54 @@ class _VmDelegate extends DatabaseDelegate { return Future.value(); } - void _runWithArgs(String statement, List args) { + Future _runWithArgs(String statement, List args) async { if (args.isEmpty) { - _db.execute(statement); + await _db.execute(statement); } else { - _db.prepare(statement) - ..execute(args) - ..close(); + final stmt = await _db.prepare(statement); + await stmt.execute(args); + await stmt.close(); } } @override - Future runCustom(String statement, List args) { - _runWithArgs(statement, args); - return Future.value(); + Future runCustom(String statement, List args) async { + await _runWithArgs(statement, args); } @override - Future runInsert(String statement, List args) { - _runWithArgs(statement, args); - return Future.value(_db.lastInsertId); + Future runInsert(String statement, List args) async { + await _runWithArgs(statement, args); + return await _db.getLastInsertId(); } @override - Future runUpdate(String statement, List args) { - _runWithArgs(statement, args); - return Future.value(_db.updatedRows); + Future runUpdate(String statement, List args) async { + await _runWithArgs(statement, args); + return await _db.getUpdatedRows(); } @override - Future runSelect(String statement, List args) { - final stmt = _db.prepare(statement); - final result = stmt.select(args); - stmt.close(); + Future runSelect(String statement, List args) async { + final stmt = await _db.prepare(statement); + final result = await stmt.select(args); + await stmt.close(); return Future.value(QueryResult(result.columnNames, result.rows)); } } class _VmVersionDelegate extends DynamicVersionDelegate { - final Database database; + final BaseDatabase database; _VmVersionDelegate(this.database); @override - Future get schemaVersion => Future.value(database.userVersion); + Future get schemaVersion => Future.value(database.userVersion()); @override - Future setSchemaVersion(int version) { - database.userVersion = version; + Future setSchemaVersion(int version) async { + await database.setUserVersion(version); return Future.value(); } } From 0d5659493349fff8897a5b4fdd5e77eaf39c70d0 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 22 Sep 2019 14:21:45 +0200 Subject: [PATCH 116/117] FFI: Custom open behavior, isolate API, docs --- moor_ffi/README.md | 106 +++++++++-- moor_ffi/example/main.dart | 27 +++ moor_ffi/example/main_async.dart | 28 +++ moor_ffi/lib/database.dart | 7 +- moor_ffi/lib/moor_ffi.dart | 2 - moor_ffi/lib/open_helper.dart | 7 + moor_ffi/lib/src/bindings/bindings.dart | 2 +- moor_ffi/lib/src/impl/isolate/background.dart | 179 ++++++++++++++++++ moor_ffi/lib/src/impl/isolate/isolate_db.dart | 105 ++++++++++ moor_ffi/lib/src/load_library.dart | 3 +- moor_ffi/pubspec.yaml | 9 +- moor_ffi/test/runners.dart | 64 +++++++ moor_ffi/test/suite/select.dart | 21 ++ moor_ffi/test/suite/user_version.dart | 24 +++ 14 files changed, 552 insertions(+), 32 deletions(-) create mode 100644 moor_ffi/example/main.dart create mode 100644 moor_ffi/example/main_async.dart create mode 100644 moor_ffi/lib/open_helper.dart create mode 100644 moor_ffi/lib/src/impl/isolate/background.dart create mode 100644 moor_ffi/lib/src/impl/isolate/isolate_db.dart create mode 100644 moor_ffi/test/runners.dart create mode 100644 moor_ffi/test/suite/select.dart create mode 100644 moor_ffi/test/suite/user_version.dart diff --git a/moor_ffi/README.md b/moor_ffi/README.md index 1ffbb693..5dfb221b 100644 --- a/moor_ffi/README.md +++ b/moor_ffi/README.md @@ -1,38 +1,102 @@ # moor_ffi -Moor backend that uses `dart:ffi`. Note that, while we have integration tests -on this package, it depends on the `dart:ffi` apis, which are in "preview" status at the moment. -Thus, this library is not suited for production use. +Experimental bindings to sqlite by using `dart:ffi`. This library contains utils to make +integration with [moor](https://pub.dev/packages/moor) easier, but it can also be used +as a standalone package. -If you want to use moor on Android or iOS, see the [getting started guide](https://moor.simonbinder.eu/docs/getting-started/) -which recommends to use the [moor_flutter](https://pub.dev/packages/moor_flutter) package. -At the moment, this library is targeted at advanced moor users who want to try out the `ffi` -backend. +## Warnings +At the moment, `dart:ffi` is in preview and there will be breaking changes that this +library has to adapt to. This library has been tested on Dart `2.5.0`. + +If you're using a development Dart version (this includes Flutter channels that are not +`stable`), this library might not work. + +If you just want to use moor, using the [moor_flutter](https://pub.dev/packages/moor_flutter) +package is the better option at the moment. ## Supported platforms -At the moment, this plugin only supports Android without further work. However, it's also going -to run on all platforms that expose `sqlite3` as a shared native library (macOS and virtually -all Linux distros, I'm not sure about Windows). Native iOS and macOS support is planned. -As Flutter desktop doesn't support plugins on Windows and Linux yet, we can't bundle the -sqlite library on those platforms. +You can make this library work on any platform that let's you obtain a `DynamicLibrary` +from which moor_ffi loads the functions (see below). + +Out of the box, this libraries supports all platforms where `sqlite3` is installed: +- iOS: Yes +- macOS: Yes +- Linux: Available on most distros +- Windows: When the user has installed sqlite (they probably have) +- Android: Yes when used with Flutter + +This library works with and without Flutter. +If you're using Flutter, this library will bundle `sqlite3` in your Android app. This +requires the Android NDK to be installed (You can get the NDK in the [SDK Manager](https://developer.android.com/studio/intro/update.html#sdk-manager) +of Android Studio). Note that the first `flutter run` is going to take a very long time as +we need to compile sqlite. + +### On other platforms +Using this library on platforms that are not supported out of the box is fairly +straightforward. For instance, if you release your own `sqlite3.so` with your application, +you could use +```dart +import 'dart:ffi'; +import 'dart:io'; +import 'package:moor_ffi/database.dart'; +import 'package:moor_ffi/open_helper.dart'; + +void main() { + open.overrideFor(OperatingSystem.linux, _openOnLinux); + + final db = Database.memory(); + db.close(); +} + +DynamicLibrary _openOnLinux() { + final script = File(Platform.script.toFilePath()); + final libraryNextToScript = File('${script.path}/sqlite3.so'); + return DynamicLibrary.open(libraryNextToScript.path); +} +``` +Just be sure to first override the behavior and then opening the database. Further, +if you want to use the isolate api, you can only use a static method or top-level +function to open the library. + +### Supported datatypes +This library supports `null`, `int`, other `num`s (converted to double), +`String` and `Uint8List` to bind args. Returned columns from select statements +will have the same types. + +## Using without moor +```dart +import 'package:moor_ffi/database.dart'; + +void main() { + final database = Database.memory(); + // run some database operations. See the example for details + database.close(); +} +``` + +You can also use an asynchronous API on a background isolate by using `IsolateDb.openFile` +or `IsolateDb.openMemory`, respectively. be aware that the asynchronous API is much slower, +but it moves work out of the UI isolate. + +Be sure to __always__ call `Database.close` to avoid memory leaks! ## Migrating from moor_flutter +__Note__: For production apps, please use `moor_flutter` until this package +reaches a stable version. + Add both `moor` and `moor_ffi` to your pubspec, the `moor_flutter` dependency can be dropped. ```yaml dependencies: - moor: ^2.0.0 + moor: ^1.7.0 moor_ffi: ^0.0.1 dev_dependencies: - moor: ^2.0.0 + moor_generator: ^1.7.0 ``` -In your main database file, replace the `package:moor_flutter/moor_flutter.dart` import with -`package:moor/moor.dart` and `package:moor_ffi/moor_ffi.dart`. +In the file where you created a `FlutterQueryExecutor`, replace the `moor_flutter` import +with both `package:moor/moor.dart` and `package:moor_ffi/moor_ffi.dart`. + In all other project files that use moor apis (e.g. a `Value` class for companions), just import `package:moor/moor.dart`. -Finally, replace usages of `FlutterQueryExecutor` with `VmDatabase`. - -## Notes -After importing this library, the first Flutter build is going to take a very long time. The reason is that we're -compiling sqlite to bundle it with your app. Subsequent builds should take an acceptable time to execute. +Finally, replace usages of `FlutterQueryExecutor` with `VmDatabase`. \ No newline at end of file diff --git a/moor_ffi/example/main.dart b/moor_ffi/example/main.dart new file mode 100644 index 00000000..afbd1708 --- /dev/null +++ b/moor_ffi/example/main.dart @@ -0,0 +1,27 @@ +import 'package:moor_ffi/database.dart'; + +const _createTable = r''' +CREATE TABLE frameworks ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL +); +'''; + +void main() { + final db = Database.memory(); + db.execute(_createTable); + + final insertStmt = db.prepare('INSERT INTO frameworks(name) VALUES (?)'); + insertStmt.execute(['Flutter']); + insertStmt.execute(['AngularDart']); + insertStmt.close(); + + final selectStmt = db.prepare('SELECT * FROM frameworks ORDER BY name'); + final result = selectStmt.select(); + for (var row in result) { + print('${row['id']}: ${row['name']}'); + } + + selectStmt.close(); + db.close(); +} diff --git a/moor_ffi/example/main_async.dart b/moor_ffi/example/main_async.dart new file mode 100644 index 00000000..9f21e7aa --- /dev/null +++ b/moor_ffi/example/main_async.dart @@ -0,0 +1,28 @@ +import 'package:moor_ffi/database.dart'; + +const _createTable = r''' +CREATE TABLE frameworks ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name VARCHAR NOT NULL +); +'''; + +void main() async { + final db = await IsolateDb.openMemory(); + await db.execute(_createTable); + + final insertStmt = + await db.prepare('INSERT INTO frameworks(name) VALUES (?)'); + await insertStmt.execute(['Flutter']); + await insertStmt.execute(['AngularDart']); + await insertStmt.close(); + + final selectStmt = await db.prepare('SELECT * FROM frameworks ORDER BY name'); + final result = await selectStmt.select(); + for (var row in result) { + print('${row['id']}: ${row['name']}'); + } + + await selectStmt.close(); + await db.close(); +} diff --git a/moor_ffi/lib/database.dart b/moor_ffi/lib/database.dart index 9b6157eb..319b150b 100644 --- a/moor_ffi/lib/database.dart +++ b/moor_ffi/lib/database.dart @@ -1,6 +1,11 @@ -/// Exports the raw [] +/// Exports the low-level [Database] and [IsolateDb] classes to run operations +/// on a sqflite database. library database; +import 'package:moor_ffi/src/bindings/types.dart'; +import 'src/impl/isolate/isolate_db.dart'; + export 'src/api/database.dart'; export 'src/api/result.dart'; export 'src/impl/database.dart' show SqliteException, Database; +export 'src/impl/isolate/isolate_db.dart'; diff --git a/moor_ffi/lib/moor_ffi.dart b/moor_ffi/lib/moor_ffi.dart index 7954e761..af2469b7 100644 --- a/moor_ffi/lib/moor_ffi.dart +++ b/moor_ffi/lib/moor_ffi.dart @@ -1,4 +1,3 @@ -import 'dart:ffi'; import 'dart:io'; import 'package:moor/backends.dart'; @@ -6,4 +5,3 @@ import 'package:moor/moor.dart'; import 'package:moor_ffi/database.dart'; part 'src/vm_database.dart'; -part 'src/load_library.dart'; diff --git a/moor_ffi/lib/open_helper.dart b/moor_ffi/lib/open_helper.dart new file mode 100644 index 00000000..ed421162 --- /dev/null +++ b/moor_ffi/lib/open_helper.dart @@ -0,0 +1,7 @@ +/// Utils to open a [DynamicLibrary] on platforms that aren't supported by +/// `moor_ffi` by default. +library open_helper; + +import 'dart:ffi'; + +export 'src/load_library.dart'; diff --git a/moor_ffi/lib/src/bindings/bindings.dart b/moor_ffi/lib/src/bindings/bindings.dart index 4b397358..4bc66081 100644 --- a/moor_ffi/lib/src/bindings/bindings.dart +++ b/moor_ffi/lib/src/bindings/bindings.dart @@ -4,7 +4,7 @@ import 'dart:ffi'; -import 'package:moor_ffi/moor_ffi.dart'; +import 'package:moor_ffi/open_helper.dart'; import '../ffi/blob.dart'; diff --git a/moor_ffi/lib/src/impl/isolate/background.dart b/moor_ffi/lib/src/impl/isolate/background.dart new file mode 100644 index 00000000..7f77b342 --- /dev/null +++ b/moor_ffi/lib/src/impl/isolate/background.dart @@ -0,0 +1,179 @@ +import 'dart:async'; +import 'dart:isolate'; + +import 'package:moor_ffi/database.dart'; +import 'package:moor_ffi/src/impl/database.dart'; + +enum IsolateCommandType { + openDatabase, + closeDatabase, + executeSqlDirectly, + prepareStatement, + getUserVersion, + setUserVersion, + getUpdatedRows, + getLastInsertId, + preparedSelect, + preparedExecute, + preparedClose +} + +class IsolateCommand { + final int requestId; + final IsolateCommandType type; + final dynamic data; + + /// If this command operates on a prepared statement, contains the id of that + /// statement as sent by the background isolate. + int preparedStatementId; + + IsolateCommand(this.requestId, this.type, this.data); +} + +class IsolateResponse { + final int requestId; + final dynamic response; + final dynamic error; + + IsolateResponse(this.requestId, this.response, this.error); +} + +/// Communicates with a background isolate over an RPC-like api. +class DbOperationProxy { + /// Stream of messages received by the background isolate. + final StreamController backgroundMsgs; + final ReceivePort _receivePort; + final Map _pendingRequests = {}; + + final SendPort send; + final Isolate isolate; + + int _currentRequestId = 0; + + DbOperationProxy( + this.backgroundMsgs, this._receivePort, this.send, this.isolate) { + backgroundMsgs.stream.listen(_handleResponse); + } + + Future sendRequest(IsolateCommandType type, dynamic data, + {int preparedStmtId}) { + final id = _currentRequestId++; + final cmd = IsolateCommand(id, type, data) + ..preparedStatementId = preparedStmtId; + final completer = Completer(); + _pendingRequests[id] = completer; + + send.send(cmd); + + return completer.future; + } + + void _handleResponse(dynamic response) { + if (response is IsolateResponse) { + final completer = _pendingRequests.remove(response.requestId); + if (response.error != null) { + completer.completeError(response.error); + } else { + completer.complete(response.response); + } + } + } + + void close() { + _receivePort.close(); + backgroundMsgs.close(); + isolate.kill(); + } + + static Future spawn() async { + final foregroundReceive = ReceivePort(); + final backgroundSend = foregroundReceive.sendPort; + final isolate = await Isolate.spawn(_entryPoint, backgroundSend, + debugName: 'moor_ffi background isolate'); + + final controller = StreamController.broadcast(); + foregroundReceive.listen(controller.add); + + final foregroundSend = await controller.stream + .firstWhere((msg) => msg is SendPort) as SendPort; + + return DbOperationProxy( + controller, foregroundReceive, foregroundSend, isolate); + } + + static void _entryPoint(SendPort backgroundSend) { + final backgroundReceive = ReceivePort(); + final foregroundSend = backgroundReceive.sendPort; + + // inform the main isolate about the created send port + backgroundSend.send(foregroundSend); + + BackgroundIsolateRunner(backgroundReceive, backgroundSend).start(); + } +} + +class BackgroundIsolateRunner { + final ReceivePort receive; + final SendPort send; + + Database db; + List stmts = []; + + BackgroundIsolateRunner(this.receive, this.send); + + void start() { + receive.listen((data) { + if (data is IsolateCommand) { + try { + final response = _handleCommand(data); + send.send(IsolateResponse(data.requestId, response, null)); + } catch (e) { + send.send(IsolateResponse(data.requestId, null, e)); + } + } + }); + } + + dynamic _handleCommand(IsolateCommand cmd) { + switch (cmd.type) { + case IsolateCommandType.openDatabase: + assert(db == null); + db = Database.open(cmd.data as String); + break; + case IsolateCommandType.closeDatabase: + db?.close(); + stmts.clear(); + db = null; + break; + case IsolateCommandType.executeSqlDirectly: + db.execute(cmd.data as String); + break; + case IsolateCommandType.prepareStatement: + final stmt = db.prepare(cmd.data as String); + stmts.add(stmt); + return stmts.length - 1; + case IsolateCommandType.getUserVersion: + return db.userVersion(); + case IsolateCommandType.setUserVersion: + final version = cmd.data as int; + db.setUserVersion(version); + break; + case IsolateCommandType.getUpdatedRows: + return db.getUpdatedRows(); + case IsolateCommandType.getLastInsertId: + return db.getLastInsertId(); + case IsolateCommandType.preparedSelect: + final stmt = stmts[cmd.preparedStatementId]; + return stmt.select(cmd.data as List); + case IsolateCommandType.preparedExecute: + final stmt = stmts[cmd.preparedStatementId]; + stmt.execute(cmd.data as List); + break; + case IsolateCommandType.preparedClose: + final index = cmd.preparedStatementId; + stmts[index].close(); + stmts.removeAt(index); + break; + } + } +} diff --git a/moor_ffi/lib/src/impl/isolate/isolate_db.dart b/moor_ffi/lib/src/impl/isolate/isolate_db.dart new file mode 100644 index 00000000..543fa56b --- /dev/null +++ b/moor_ffi/lib/src/impl/isolate/isolate_db.dart @@ -0,0 +1,105 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:moor_ffi/database.dart'; +import 'package:moor_ffi/src/impl/database.dart'; +import 'package:moor_ffi/src/impl/isolate/background.dart'; + +class IsolateDb implements BaseDatabase { + /// Spawns a background isolate and opens the [file] on that isolate. The file + /// will be created if it doesn't exist. + static Future openFile(File file) => open(file.absolute.path); + + /// Opens a in-memory database on a background isolates. + /// + /// If you're not using extensive queries, a synchronous [Database] will + /// provide better performance for in-memory databases! + static Future openMemory() => open(':memory:'); + + /// Spawns a background isolate and opens a sqlite3 database from its + /// filename. + static Future open(String path) async { + final proxy = await DbOperationProxy.spawn(); + + final isolate = IsolateDb._(proxy); + await isolate._open(path); + + return isolate; + } + + final DbOperationProxy _proxy; + IsolateDb._(this._proxy); + + Future _sendAndAssumeInt(IsolateCommandType type, [dynamic data]) async { + return await _proxy.sendRequest(type, data) as int; + } + + Future _open(String path) { + return _proxy.sendRequest(IsolateCommandType.openDatabase, path); + } + + @override + Future close() async { + await _proxy.sendRequest(IsolateCommandType.closeDatabase, null); + _proxy.close(); + } + + @override + Future execute(String sql) async { + await _proxy.sendRequest(IsolateCommandType.executeSqlDirectly, sql); + } + + @override + Future getLastInsertId() async { + return _sendAndAssumeInt(IsolateCommandType.getLastInsertId); + } + + @override + Future getUpdatedRows() async { + return _sendAndAssumeInt(IsolateCommandType.getUpdatedRows); + } + + @override + FutureOr prepare(String sql) async { + final id = + await _sendAndAssumeInt(IsolateCommandType.prepareStatement, sql); + return IsolatePreparedStatement(this, id); + } + + @override + Future setUserVersion(int version) async { + await _proxy.sendRequest(IsolateCommandType.setUserVersion, version); + } + + @override + Future userVersion() async { + return _sendAndAssumeInt(IsolateCommandType.getUserVersion); + } +} + +class IsolatePreparedStatement implements BasePreparedStatement { + final IsolateDb _db; + final int _id; + + IsolatePreparedStatement(this._db, this._id); + + @override + Future close() async { + await _db._proxy.sendRequest(IsolateCommandType.preparedClose, null, + preparedStmtId: _id); + } + + @override + Future execute([List params]) async { + await _db._proxy.sendRequest(IsolateCommandType.preparedExecute, params, + preparedStmtId: _id); + } + + @override + Future select([List params]) async { + final response = await _db._proxy.sendRequest( + IsolateCommandType.preparedSelect, params, + preparedStmtId: _id); + return response as Result; + } +} diff --git a/moor_ffi/lib/src/load_library.dart b/moor_ffi/lib/src/load_library.dart index db3108e0..433526fc 100644 --- a/moor_ffi/lib/src/load_library.dart +++ b/moor_ffi/lib/src/load_library.dart @@ -1,4 +1,5 @@ -part of 'package:moor_ffi/moor_ffi.dart'; +import 'dart:ffi'; +import 'dart:io'; /// Signature responsible for loading the dynamic sqlite3 library that moor will /// use. diff --git a/moor_ffi/pubspec.yaml b/moor_ffi/pubspec.yaml index 26ba7882..8536dcf0 100644 --- a/moor_ffi/pubspec.yaml +++ b/moor_ffi/pubspec.yaml @@ -8,19 +8,16 @@ environment: sdk: ">=2.5.0-dev <2.6.0" dependencies: - moor: ^2.0.0 + moor: ">=1.7.0 <2.1.0" # flutter: # sdk: flutter dev_dependencies: test: ^1.6.0 + path: ^1.6.0 flutter: plugin: # the flutter.plugin key needs to exists so that this project gets recognized as a plugin when imported. We need to # get recognized as a plugin so that our build scripts are executed. - foo: bar - -dependency_overrides: - moor: - path: ../moor \ No newline at end of file + foo: bar \ No newline at end of file diff --git a/moor_ffi/test/runners.dart b/moor_ffi/test/runners.dart new file mode 100644 index 00000000..279df371 --- /dev/null +++ b/moor_ffi/test/runners.dart @@ -0,0 +1,64 @@ +import 'dart:async'; +import 'dart:io'; + +import 'package:path/path.dart' as p; +import 'package:test/test.dart'; +import 'package:moor_ffi/database.dart'; + +import 'suite/select.dart' as select; +import 'suite/user_version.dart' as user_version; + +var _tempFileCounter = 0; +List _createdFiles = []; +File temporaryFile() { + final count = _tempFileCounter++; + final path = + p.join(Directory.systemTemp.absolute.path, 'moor_ffi_test_$count.db'); + final file = File(path); + _createdFiles.add(file); + return file; +} + +abstract class TestedDatabase { + FutureOr openFile(File file); + FutureOr openMemory(); +} + +class TestRegularDatabase implements TestedDatabase { + @override + BaseDatabase openFile(File file) => Database.openFile(file); + + @override + BaseDatabase openMemory() => Database.memory(); +} + +class TestIsolateDatabase implements TestedDatabase { + @override + Future openFile(File file) => IsolateDb.openFile(file); + + @override + FutureOr openMemory() => IsolateDb.openMemory(); +} + +void main() { + group('regular database', () { + _declareAll(TestRegularDatabase()); + }); + + group('isolate database', () { + _declareAll(TestIsolateDatabase()); + }); + + tearDownAll(() async { + for (var file in _createdFiles) { + if (await file.exists()) { + await file.delete(); + } + } + }); +} + +void _declareAll(TestedDatabase db) { + select.main(db); + user_version.main(db); +} diff --git a/moor_ffi/test/suite/select.dart b/moor_ffi/test/suite/select.dart new file mode 100644 index 00000000..a4719c09 --- /dev/null +++ b/moor_ffi/test/suite/select.dart @@ -0,0 +1,21 @@ +import 'package:test/test.dart'; + +import '../runners.dart'; + +void main(TestedDatabase db) { + test('select statements return expected value', () async { + final opened = await db.openMemory(); + + final prepared = await opened.prepare('SELECT ?'); + + final result1 = await prepared.select([1]); + expect(result1.columnNames, ['?']); + expect(result1.single.columnAt(0), 1); + + final result2 = await prepared.select([2]); + expect(result2.columnNames, ['?']); + expect(result2.single.columnAt(0), 2); + + await opened.close(); + }); +} diff --git a/moor_ffi/test/suite/user_version.dart b/moor_ffi/test/suite/user_version.dart new file mode 100644 index 00000000..cf07277c --- /dev/null +++ b/moor_ffi/test/suite/user_version.dart @@ -0,0 +1,24 @@ +import 'package:test/test.dart'; + +import '../runners.dart'; + +void main(TestedDatabase db) { + test('can set the user version on a database', () async { + final file = temporaryFile(); + final opened = await db.openFile(file); + + var version = await opened.userVersion(); + expect(version, 0); + + await opened.setUserVersion(3); + version = await opened.userVersion(); + expect(version, 3); + + // ensure that the version is stored on file + await opened.close(); + + final another = await db.openFile(file); + expect(await another.userVersion(), 3); + await another.close(); + }); +} From b99bc820dae64e14c771e2d04f170f8f869894f3 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 22 Sep 2019 16:26:17 +0200 Subject: [PATCH 117/117] Add more tests in moor_ffi subproject --- moor_ffi/README.md | 6 ++- moor_ffi/lib/src/impl/database.dart | 18 +++++--- moor_ffi/lib/src/impl/isolate/background.dart | 17 ++++++- moor_ffi/lib/src/impl/prepared_statement.dart | 2 +- moor_ffi/pubspec.yaml | 9 ++-- moor_ffi/test/ffi/blob_test.dart | 21 +++++++++ moor_ffi/test/runners.dart | 4 ++ moor_ffi/test/suite/insert.dart | 18 ++++++++ moor_ffi/test/suite/prepared_statements.dart | 44 +++++++++++++++++++ 9 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 moor_ffi/test/ffi/blob_test.dart create mode 100644 moor_ffi/test/suite/insert.dart create mode 100644 moor_ffi/test/suite/prepared_statements.dart diff --git a/moor_ffi/README.md b/moor_ffi/README.md index 5dfb221b..6e392297 100644 --- a/moor_ffi/README.md +++ b/moor_ffi/README.md @@ -96,7 +96,9 @@ dev_dependencies: In the file where you created a `FlutterQueryExecutor`, replace the `moor_flutter` import with both `package:moor/moor.dart` and `package:moor_ffi/moor_ffi.dart`. - In all other project files that use moor apis (e.g. a `Value` class for companions), just import `package:moor/moor.dart`. -Finally, replace usages of `FlutterQueryExecutor` with `VmDatabase`. \ No newline at end of file +Finally, replace usages of `FlutterQueryExecutor` with `VmDatabase`. + +Note that, at the moment, there is no counterpart for `FlutterQueryExecutor.inDatabasePath` and that the async API using +a background isolate is not available yet. Both shortcomings with be fixed by the upcoming moor 2.0 release. \ No newline at end of file diff --git a/moor_ffi/lib/src/impl/database.dart b/moor_ffi/lib/src/impl/database.dart index 1878013b..2d06e084 100644 --- a/moor_ffi/lib/src/impl/database.dart +++ b/moor_ffi/lib/src/impl/database.dart @@ -44,6 +44,7 @@ class Database implements BaseDatabase { if (resultCode == Errors.SQLITE_OK) { return Database._(dbPointer); } else { + bindings.sqlite3_close_v2(dbPointer); throw SqliteException._fromErrorCode(dbPointer, resultCode); } } @@ -56,6 +57,12 @@ class Database implements BaseDatabase { @override void close() { + // close all prepared statements first + _isClosed = true; + for (var stmt in _preparedStmt) { + stmt.close(); + } + final code = bindings.sqlite3_close_v2(_db); SqliteException exception; if (code != Errors.SQLITE_OK) { @@ -63,10 +70,7 @@ class Database implements BaseDatabase { } _isClosed = true; - for (var stmt in _preparedStmt) { - stmt.close(); - } - _db.free(); + // we don't need to deallocate the _db pointer, sqlite takes care of that if (exception != null) { throw exception; @@ -82,6 +86,7 @@ class Database implements BaseDatabase { @override void execute(String sql) { _ensureOpen(); + final sqlPtr = CBlob.allocateString(sql); final errorOut = Pointer>.allocate(); @@ -125,7 +130,10 @@ class Database implements BaseDatabase { throw SqliteException._fromErrorCode(_db, resultCode); } - return PreparedStatement._(stmt, this); + final prepared = PreparedStatement._(stmt, this); + _preparedStmt.add(prepared); + + return prepared; } @override diff --git a/moor_ffi/lib/src/impl/isolate/background.dart b/moor_ffi/lib/src/impl/isolate/background.dart index 7f77b342..7ef7f457 100644 --- a/moor_ffi/lib/src/impl/isolate/background.dart +++ b/moor_ffi/lib/src/impl/isolate/background.dart @@ -48,6 +48,8 @@ class DbOperationProxy { final SendPort send; final Isolate isolate; + var closed = false; + int _currentRequestId = 0; DbOperationProxy( @@ -57,6 +59,10 @@ class DbOperationProxy { Future sendRequest(IsolateCommandType type, dynamic data, {int preparedStmtId}) { + if (closed) { + throw StateError('Tried to call a database method after .close()'); + } + final id = _currentRequestId++; final cmd = IsolateCommand(id, type, data) ..preparedStatementId = preparedStmtId; @@ -80,6 +86,7 @@ class DbOperationProxy { } void close() { + closed = true; _receivePort.close(); backgroundMsgs.close(); isolate.kill(); @@ -128,7 +135,15 @@ class BackgroundIsolateRunner { final response = _handleCommand(data); send.send(IsolateResponse(data.requestId, response, null)); } catch (e) { - send.send(IsolateResponse(data.requestId, null, e)); + if (e is Error) { + // errors contain a StackTrace, which cannot be sent. So we just + // send the description of that stacktrace. + final exception = + Exception('Error in background isolate: $e\n${e.stackTrace}'); + send.send(IsolateResponse(data.requestId, null, exception)); + } else { + send.send(IsolateResponse(data.requestId, null, e)); + } } } }); diff --git a/moor_ffi/lib/src/impl/prepared_statement.dart b/moor_ffi/lib/src/impl/prepared_statement.dart index 01ff6ffa..37fd8dc8 100644 --- a/moor_ffi/lib/src/impl/prepared_statement.dart +++ b/moor_ffi/lib/src/impl/prepared_statement.dart @@ -22,7 +22,7 @@ class PreparedStatement implements BasePreparedStatement { void _ensureNotFinalized() { if (_closed) { - throw Exception('Tried to operate on a released prepared statement'); + throw StateError('Tried to operate on a released prepared statement'); } } diff --git a/moor_ffi/pubspec.yaml b/moor_ffi/pubspec.yaml index 8536dcf0..8f290249 100644 --- a/moor_ffi/pubspec.yaml +++ b/moor_ffi/pubspec.yaml @@ -1,16 +1,15 @@ name: moor_ffi -description: "Experimental moor implementation that uses dart:ffi" +description: "Experimental sqlite bindings using dart:ffi" version: 0.0.1 -author: -homepage: +author: Simon Binder +homepage: https://github.com/simolus3/moor/tree/develop/moor_ffi +issue_tracker: https://github.com/simolus3/moor/issues environment: sdk: ">=2.5.0-dev <2.6.0" dependencies: moor: ">=1.7.0 <2.1.0" -# flutter: -# sdk: flutter dev_dependencies: test: ^1.6.0 diff --git a/moor_ffi/test/ffi/blob_test.dart b/moor_ffi/test/ffi/blob_test.dart new file mode 100644 index 00000000..ab3612aa --- /dev/null +++ b/moor_ffi/test/ffi/blob_test.dart @@ -0,0 +1,21 @@ +import 'package:moor/moor.dart'; +import 'package:moor_ffi/src/ffi/blob.dart'; +import 'package:test/test.dart'; + +void main() { + test('utf8 store and load test', () { + final content = 'Hasta MaƱana'; + final blob = CBlob.allocateString(content); + + expect(blob.load().readString(), content); + blob.free(); + }); + + test('blob load and store test', () { + final data = List.generate(256, (x) => x); + final blob = CBlob.allocate(Uint8List.fromList(data)); + + expect(blob.load().read(256), data); + blob.free(); + }); +} diff --git a/moor_ffi/test/runners.dart b/moor_ffi/test/runners.dart index 279df371..084c29cf 100644 --- a/moor_ffi/test/runners.dart +++ b/moor_ffi/test/runners.dart @@ -5,6 +5,8 @@ import 'package:path/path.dart' as p; import 'package:test/test.dart'; import 'package:moor_ffi/database.dart'; +import 'suite/insert.dart' as insert; +import 'suite/prepared_statements.dart' as prepared_statements; import 'suite/select.dart' as select; import 'suite/user_version.dart' as user_version; @@ -59,6 +61,8 @@ void main() { } void _declareAll(TestedDatabase db) { + insert.main(db); + prepared_statements.main(db); select.main(db); user_version.main(db); } diff --git a/moor_ffi/test/suite/insert.dart b/moor_ffi/test/suite/insert.dart new file mode 100644 index 00000000..264ad14b --- /dev/null +++ b/moor_ffi/test/suite/insert.dart @@ -0,0 +1,18 @@ +import 'package:test/test.dart'; + +import '../runners.dart'; + +void main(TestedDatabase db) { + test('insert statements report their id', () async { + final opened = await db.openMemory(); + await opened + .execute('CREATE TABLE tbl(a INTEGER PRIMARY KEY AUTOINCREMENT)'); + + for (var i = 0; i < 5; i++) { + await opened.execute('INSERT INTO tbl DEFAULT VALUES'); + expect(await opened.getLastInsertId(), i + 1); + } + + await opened.close(); + }); +} diff --git a/moor_ffi/test/suite/prepared_statements.dart b/moor_ffi/test/suite/prepared_statements.dart new file mode 100644 index 00000000..051269b9 --- /dev/null +++ b/moor_ffi/test/suite/prepared_statements.dart @@ -0,0 +1,44 @@ +import 'package:test/test.dart'; + +import '../runners.dart'; + +void main(TestedDatabase db) { + test('prepared statements can be used multiple times', () async { + final opened = await db.openMemory(); + await opened.execute('CREATE TABLE tbl (a TEXT);'); + + final stmt = await opened.prepare('INSERT INTO tbl(a) VALUES(?)'); + await stmt.execute(['a']); + await stmt.execute(['b']); + await stmt.close(); + + final select = await opened.prepare('SELECT * FROM tbl ORDER BY a'); + final result = await select.select(); + + expect(result, hasLength(2)); + expect(result.map((row) => row['a']), ['a', 'b']); + + await select.close(); + + await opened.close(); + }); + + test('prepared statements cannot be used after close', () async { + final opened = await db.openMemory(); + + final stmt = await opened.prepare('SELECT ?'); + await stmt.close(); + + expect(stmt.select, throwsA(anything)); + + await opened.close(); + }); + + test('prepared statements cannot be used after db is closed', () async { + final opened = await db.openMemory(); + final stmt = await opened.prepare('SELECT 1'); + await opened.close(); + + expect(stmt.select, throwsA(anything)); + }); +}