diff --git a/drift/lib/src/runtime/api/connection_user.dart b/drift/lib/src/runtime/api/connection_user.dart index 9d0c562a..66d194b2 100644 --- a/drift/lib/src/runtime/api/connection_user.dart +++ b/drift/lib/src/runtime/api/connection_user.dart @@ -606,3 +606,15 @@ extension on TransactionExecutor { } } } + +/// Exposes the private `_runConnectionZoned` method for other parts of drift. +/// +/// This is only used by the DevTools extension. +@internal +extension RunWithEngine on DatabaseConnectionUser { + /// Call the private [_runConnectionZoned] method. + Future runConnectionZoned( + DatabaseConnectionUser user, Future Function() calculation) { + return _runConnectionZoned(user, calculation); + } +} diff --git a/drift/lib/src/runtime/devtools/service_extension.dart b/drift/lib/src/runtime/devtools/service_extension.dart index 1b4d332b..4b1beb11 100644 --- a/drift/lib/src/runtime/devtools/service_extension.dart +++ b/drift/lib/src/runtime/devtools/service_extension.dart @@ -2,9 +2,10 @@ import 'dart:async'; import 'dart:convert'; import 'dart:developer'; +import 'package:drift/drift.dart'; import 'package:drift/src/remote/protocol.dart'; +import 'package:drift/src/runtime/executor/transactions.dart'; -import '../query_builder/query_builder.dart'; import 'devtools.dart'; /// A service extension making asynchronous requests on drift databases @@ -26,7 +27,7 @@ class DriftServiceExtension { final stream = tracked.database.tableUpdates(); final id = _subscriptionId++; - stream.listen((event) { + _activeSubscriptions[id] = stream.listen((event) { postEvent('event', { 'subscription': id, 'payload': @@ -60,6 +61,16 @@ class DriftServiceExtension { }; return _protocol.encodePayload(result); + case 'collect-expected-schema': + final executor = _CollectCreateStatements(); + await tracked.database.runConnectionZoned( + BeforeOpenRunner(tracked.database, executor), () async { + // ignore: invalid_use_of_protected_member, invalid_use_of_visible_for_testing_member + final migrator = tracked.database.createMigrator(); + await migrator.createAll(); + }); + + return executor.statements; default: throw UnsupportedError('Method $action'); } @@ -96,3 +107,52 @@ class DriftServiceExtension { static const _protocol = DriftProtocol(); } + +class _CollectCreateStatements extends QueryExecutor { + final List statements = []; + + @override + TransactionExecutor beginTransaction() { + throw UnimplementedError(); + } + + @override + SqlDialect get dialect => SqlDialect.sqlite; + + @override + Future ensureOpen(QueryExecutorUser user) { + return Future.value(true); + } + + @override + Future runBatched(BatchedStatements statements) { + throw UnimplementedError(); + } + + @override + Future runCustom(String statement, [List? args]) { + statements.add(statement); + return Future.value(); + } + + @override + Future runDelete(String statement, List args) { + throw UnimplementedError(); + } + + @override + Future runInsert(String statement, List args) { + throw UnimplementedError(); + } + + @override + Future>> runSelect( + String statement, List args) { + throw UnimplementedError(); + } + + @override + Future runUpdate(String statement, List args) { + throw UnimplementedError(); + } +} diff --git a/drift/lib/src/runtime/devtools/shared.dart b/drift/lib/src/runtime/devtools/shared.dart index c0757cac..029c1ff4 100644 --- a/drift/lib/src/runtime/devtools/shared.dart +++ b/drift/lib/src/runtime/devtools/shared.dart @@ -93,6 +93,7 @@ class EntityDescription { return EntityDescription( name: entity.entityName, type: switch (entity) { + VirtualTableInfo() => 'virtual_table', TableInfo() => 'table', ViewInfo() => 'view', Index() => 'index', diff --git a/drift/pubspec.yaml b/drift/pubspec.yaml index 1280fc96..fa0472b8 100644 --- a/drift/pubspec.yaml +++ b/drift/pubspec.yaml @@ -37,3 +37,4 @@ dev_dependencies: shelf: ^1.3.0 stack_trace: ^1.10.0 test_descriptor: ^2.0.1 + vm_service: ^13.0.0 diff --git a/drift/test/integration_tests/devtools/app.dart b/drift/test/integration_tests/devtools/app.dart new file mode 100644 index 00000000..48e5a7c1 --- /dev/null +++ b/drift/test/integration_tests/devtools/app.dart @@ -0,0 +1,11 @@ +import 'package:drift/native.dart'; + +import '../../generated/todos.dart'; + +void main() { + TodoDb(NativeDatabase.memory()); + print('database created'); + + // Keep the process alive + Stream.periodic(const Duration(seconds: 10)).listen(null); +} diff --git a/drift/test/integration_tests/devtools/devtools_test.dart b/drift/test/integration_tests/devtools/devtools_test.dart new file mode 100644 index 00000000..b198ce1e --- /dev/null +++ b/drift/test/integration_tests/devtools/devtools_test.dart @@ -0,0 +1,73 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:test/test.dart'; +import 'package:vm_service/vm_service.dart'; +import 'package:path/path.dart' as p; +import 'package:vm_service/vm_service_io.dart'; + +void main() { + late Process child; + late VmService vm; + late String isolateId; + + setUpAll(() async { + final socket = await ServerSocket.bind(InternetAddress.loopbackIPv4, 0); + final port = socket.port; + await socket.close(); + + String sdk = p.dirname(p.dirname(Platform.resolvedExecutable)); + child = await Process.start(p.join(sdk, 'bin', 'dart'), [ + 'run', + '--enable-vm-service=$port', + '--disable-service-auth-codes', + '--enable-asserts', + 'test/integration_tests/devtools/app.dart', + ]); + + final vmServiceListening = Completer(); + final databaseOpened = Completer(); + + child.stdout + .map(utf8.decode) + .transform(const LineSplitter()) + .listen((line) { + if (line.startsWith('The Dart VM service is listening')) { + vmServiceListening.complete(); + } else if (line == 'database created') { + databaseOpened.complete(); + } else if (!line.startsWith('The Dart DevTools')) { + print('[child]: $line'); + } + }); + + await vmServiceListening.future; + vm = await vmServiceConnectUri('ws://localhost:$port/ws'); + + final state = await vm.getVM(); + isolateId = state.isolates!.single.id!; + + await databaseOpened.future; + }); + + tearDownAll(() async { + child.kill(); + }); + + test('can list create statements', () async { + final response = await vm.callServiceExtension( + 'ext.drift.database', + args: {'action': 'collect-expected-schema', 'db': '0'}, + isolateId: isolateId, + ); + + expect( + response.json!['r'], + containsAll([ + 'CREATE TABLE IF NOT EXISTS "categories" ("id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "desc" TEXT NOT NULL UNIQUE, "priority" INTEGER NOT NULL DEFAULT 0, "description_in_upper_case" TEXT NOT NULL GENERATED ALWAYS AS (UPPER("desc")) VIRTUAL);', + 'CREATE TABLE IF NOT EXISTS "todos" ("id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "title" TEXT NULL, "content" TEXT NOT NULL, "target_date" INTEGER NULL UNIQUE, "category" INTEGER NULL REFERENCES categories (id), "status" TEXT NULL, UNIQUE ("title", "category"), UNIQUE ("title", "target_date"));', + 'CREATE TABLE IF NOT EXISTS "shared_todos" ("todo" INTEGER NOT NULL, "user" INTEGER NOT NULL, PRIMARY KEY ("todo", "user"), FOREIGN KEY (todo) REFERENCES todos(id), FOREIGN KEY (user) REFERENCES users(id));' + ])); + }); +} diff --git a/drift_dev/lib/api/migrations.dart b/drift_dev/lib/api/migrations.dart index b11f2555..a09cdf15 100644 --- a/drift_dev/lib/api/migrations.dart +++ b/drift_dev/lib/api/migrations.dart @@ -2,10 +2,13 @@ import 'package:drift/drift.dart'; import 'package:drift/internal/migrations.dart'; import 'package:drift/native.dart'; import 'package:drift_dev/src/services/schema/verifier_impl.dart'; +import 'package:drift_dev/src/services/schema/verifier_common.dart'; import 'package:meta/meta.dart'; import 'package:sqlite3/sqlite3.dart'; export 'package:drift/internal/migrations.dart'; +export 'package:drift_dev/src/services/schema/verifier_common.dart' + show SchemaMismatch; abstract class SchemaVerifier { factory SchemaVerifier(SchemaInstantiationHelper helper) = @@ -130,18 +133,6 @@ class _GenerateFromScratch extends GeneratedDatabase { int get schemaVersion => 1; } -/// Thrown when the actual schema differs from the expected schema. -class SchemaMismatch implements Exception { - final String explanation; - - SchemaMismatch(this.explanation); - - @override - String toString() { - return 'Schema does not match\n$explanation'; - } -} - /// Contains an initialized schema with all tables, views, triggers and indices. /// /// You can use the [newConnection] for your database class and the diff --git a/drift_dev/lib/src/services/schema/sqlite_to_drift.dart b/drift_dev/lib/src/services/schema/sqlite_to_drift.dart index 753a2e8c..b635e82f 100644 --- a/drift_dev/lib/src/services/schema/sqlite_to_drift.dart +++ b/drift_dev/lib/src/services/schema/sqlite_to_drift.dart @@ -1,6 +1,5 @@ import 'package:analyzer/dart/element/element.dart'; import 'package:drift_dev/src/analysis/options.dart'; -import 'package:drift_dev/src/services/schema/verifier_impl.dart'; import 'package:logging/logging.dart'; import 'package:sqlite3/common.dart'; import 'package:sqlparser/sqlparser.dart'; @@ -8,6 +7,7 @@ import 'package:sqlparser/sqlparser.dart'; import '../../analysis/backend.dart'; import '../../analysis/driver/driver.dart'; import '../../analysis/results/results.dart'; +import 'verifier_common.dart'; /// Extracts drift elements from the schema of an existing database. /// diff --git a/drift_dev/lib/src/services/schema/verifier_common.dart b/drift_dev/lib/src/services/schema/verifier_common.dart new file mode 100644 index 00000000..1e734619 --- /dev/null +++ b/drift_dev/lib/src/services/schema/verifier_common.dart @@ -0,0 +1,39 @@ +import 'find_differences.dart'; + +/// Attempts to recognize whether [name] is likely the name of an internal +/// sqlite3 table (like `sqlite3_sequence`) that we should not consider when +/// comparing schemas. +bool isInternalElement(String name, List virtualTables) { + // Skip sqlite-internal tables, https://www.sqlite.org/fileformat2.html#intschema + if (name.startsWith('sqlite_')) return true; + if (virtualTables.any((v) => name.startsWith('${v}_'))) return true; + + // This file is added on some Android versions when using the native Android + // database APIs, https://github.com/simolus3/drift/discussions/2042 + if (name == 'android_metadata') return true; + + return false; +} + +void verify(List referenceSchema, List actualSchema, + bool validateDropped) { + final result = + FindSchemaDifferences(referenceSchema, actualSchema, validateDropped) + .compare(); + + if (!result.noChanges) { + throw SchemaMismatch(result.describe()); + } +} + +/// Thrown when the actual schema differs from the expected schema. +class SchemaMismatch implements Exception { + final String explanation; + + SchemaMismatch(this.explanation); + + @override + String toString() { + return 'Schema does not match\n$explanation'; + } +} diff --git a/drift_dev/lib/src/services/schema/verifier_impl.dart b/drift_dev/lib/src/services/schema/verifier_impl.dart index 181b0528..ba8b4941 100644 --- a/drift_dev/lib/src/services/schema/verifier_impl.dart +++ b/drift_dev/lib/src/services/schema/verifier_impl.dart @@ -6,6 +6,7 @@ import 'package:drift_dev/api/migrations.dart'; import 'package:sqlite3/sqlite3.dart'; import 'find_differences.dart'; +import 'verifier_common.dart'; Expando> expectedSchema = Expando(); @@ -94,21 +95,6 @@ Input? _parseInputFromSchemaRow( return Input(name, row['sql'] as String); } -/// Attempts to recognize whether [name] is likely the name of an internal -/// sqlite3 table (like `sqlite3_sequence`) that we should not consider when -/// comparing schemas. -bool isInternalElement(String name, List virtualTables) { - // Skip sqlite-internal tables, https://www.sqlite.org/fileformat2.html#intschema - if (name.startsWith('sqlite_')) return true; - if (virtualTables.any((v) => name.startsWith('${v}_'))) return true; - - // This file is added on some Android versions when using the native Android - // database APIs, https://github.com/simolus3/drift/discussions/2042 - if (name == 'android_metadata') return true; - - return false; -} - extension CollectSchemaDb on DatabaseConnectionUser { Future> collectSchemaInput(List virtualTables) async { final result = await customSelect('SELECT * FROM sqlite_master;').get(); @@ -141,17 +127,6 @@ extension CollectSchema on QueryExecutor { } } -void verify(List referenceSchema, List actualSchema, - bool validateDropped) { - final result = - FindSchemaDifferences(referenceSchema, actualSchema, validateDropped) - .compare(); - - if (!result.noChanges) { - throw SchemaMismatch(result.describe()); - } -} - class _DelegatingUser extends QueryExecutorUser { @override final int schemaVersion; diff --git a/extras/drift_devtools_extension/lib/src/db_viewer/database.dart b/extras/drift_devtools_extension/lib/src/db_viewer/database.dart index 79a6cf57..b579e76e 100644 --- a/extras/drift_devtools_extension/lib/src/db_viewer/database.dart +++ b/extras/drift_devtools_extension/lib/src/db_viewer/database.dart @@ -52,7 +52,10 @@ class ViewerDatabase implements DbViewerDatabase { @override List get entityNames => [ for (final entity in database.description.entities) - if (entity.type == 'table') entity.name, + if (entity.type == 'table' || + entity.type == 'virtual_table' || + entity.type == 'view') + entity.name, ]; @override diff --git a/extras/drift_devtools_extension/lib/src/details.dart b/extras/drift_devtools_extension/lib/src/details.dart index 7b0c0ed1..30b6f2fe 100644 --- a/extras/drift_devtools_extension/lib/src/details.dart +++ b/extras/drift_devtools_extension/lib/src/details.dart @@ -1,4 +1,5 @@ import 'package:devtools_app_shared/service.dart'; +import 'package:drift_devtools_extension/src/schema_validator.dart'; import 'package:flutter/material.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; @@ -55,6 +56,10 @@ class _DatabaseDetailsState extends ConsumerState { child: ListView( controller: controller, children: [ + const Padding( + padding: EdgeInsets.all(8), + child: DatabaseSchemaCheck(), + ), Padding( padding: const EdgeInsets.all(8.0), child: Column( diff --git a/extras/drift_devtools_extension/lib/src/remote_database.dart b/extras/drift_devtools_extension/lib/src/remote_database.dart index 38cb904d..5a2fb045 100644 --- a/extras/drift_devtools_extension/lib/src/remote_database.dart +++ b/extras/drift_devtools_extension/lib/src/remote_database.dart @@ -71,6 +71,11 @@ class RemoteDatabase { await _executeQuery(ExecuteQuery(StatementMethod.custom, sql, args)); } + Future> get createStatements async { + final res = await _driftRequest('collect-expected-schema'); + return (res as List).cast(); + } + Future _newTableSubscription() async { final result = await _driftRequest('subscribe-to-tables'); return result as int; diff --git a/extras/drift_devtools_extension/lib/src/schema_validator.dart b/extras/drift_devtools_extension/lib/src/schema_validator.dart new file mode 100644 index 00000000..cd46d74f --- /dev/null +++ b/extras/drift_devtools_extension/lib/src/schema_validator.dart @@ -0,0 +1,184 @@ +import 'package:devtools_app_shared/ui.dart'; +// ignore: implementation_imports +import 'package:drift_dev/src/services/schema/find_differences.dart'; +// ignore: implementation_imports +import 'package:drift_dev/src/services/schema/verifier_common.dart'; +import 'package:flutter/gestures.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_riverpod/flutter_riverpod.dart'; +import 'package:sqlite3/wasm.dart' hide Row; +import 'package:url_launcher/url_launcher.dart'; + +import 'details.dart'; +import 'remote_database.dart'; +import 'service.dart'; + +sealed class SchemaStatus {} + +final class DidNotValidateYet implements SchemaStatus { + const DidNotValidateYet(); +} + +final class SchemaComparisonResult implements SchemaStatus { + final bool schemaValid; + final String message; + + SchemaComparisonResult({required this.schemaValid, required this.message}); +} + +final schemaStateProvider = + AsyncNotifierProvider.autoDispose( + SchemaVerifier._); + +class SchemaVerifier extends AutoDisposeAsyncNotifier { + RemoteDatabase? _database; + CommonSqlite3? _sqlite3; + + SchemaVerifier._(); + + @override + Future build() async { + _database = await ref.read(loadedDatabase.future); + _sqlite3 = await ref.read(sqliteProvider.future); + + return const DidNotValidateYet(); + } + + Future validate() async { + state = const AsyncLoading(); + state = await AsyncValue.guard(() async { + final database = _database!; + + final virtualTables = database.description.entities + .where((e) => e.type == 'virtual_table') + .map((e) => e.name) + .toList(); + + final expected = await _inputFromNewDatabase(virtualTables); + final actual = []; + + for (final row in await database + .select('SELECT name, sql FROM sqlite_schema;', [])) { + final name = row['name'] as String; + final sql = row['sql'] as String; + + if (!isInternalElement(name, virtualTables)) { + actual.add(Input(name, sql)); + } + } + + try { + verify(expected, actual, true); + return SchemaComparisonResult( + schemaValid: true, + message: 'The schema of the database matches its Dart and .drift ' + 'definitions, meaning that migrations are likely correct.', + ); + } on SchemaMismatch catch (e) { + return SchemaComparisonResult( + schemaValid: false, + message: e.toString(), + ); + } + }); + } + + Future> _inputFromNewDatabase(List virtuals) async { + final expectedStatements = await _database!.createStatements; + final newDatabase = _sqlite3!.openInMemory(); + final inputs = []; + + for (var statement in expectedStatements) { + newDatabase.execute(statement); + } + + for (final row + in newDatabase.select('SELECT name, sql FROM sqlite_schema;', [])) { + final name = row['name'] as String; + final sql = row['sql'] as String; + + if (!isInternalElement(name, virtuals)) { + inputs.add(Input(name, sql)); + } + } + + newDatabase.dispose(); + return inputs; + } +} + +class DatabaseSchemaCheck extends ConsumerWidget { + const DatabaseSchemaCheck({super.key}); + + @override + Widget build(BuildContext context, WidgetRef ref) { + final state = ref.watch(schemaStateProvider); + + final description = switch (state) { + AsyncData( + value: SchemaComparisonResult(schemaValid: true, :var message) + ) => + Text.rich(TextSpan( + children: [ + const TextSpan( + text: 'Success! ', style: TextStyle(color: Colors.green)), + TextSpan(text: message), + ], + )), + AsyncData( + value: SchemaComparisonResult(schemaValid: false, :var message) + ) => + Text.rich(TextSpan( + children: [ + const TextSpan( + text: 'Mismatch detected! ', + style: TextStyle(color: Colors.red)), + TextSpan(text: message), + ], + )), + AsyncError(:var error) => + Text('The schema could not be validated due to an error: $error'), + _ => Text.rich(TextSpan( + text: 'By validating your schema, you can ensure that the current ' + 'state of the database in your app (after migrations ran) ' + 'matches the expected state of tables as defined in your sources. ', + children: [ + TextSpan( + text: 'Learn more', + style: const TextStyle( + decoration: TextDecoration.underline, + ), + recognizer: TapGestureRecognizer() + ..onTap = () async { + await launchUrl(Uri.parse( + 'https://drift.simonbinder.eu/docs/migrations/#verifying-a-database-schema-at-runtime')); + }, + ), + ], + )), + }; + + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Padding( + padding: const EdgeInsets.only(bottom: 8), + child: description, + ), + DevToolsButton( + label: switch (state) { + AsyncError() || + AsyncData(value: SchemaComparisonResult()) => + 'Validate again', + _ => 'Validate schema', + }, + onPressed: () { + if (state is! AsyncLoading) { + ref.read(schemaStateProvider.notifier).validate(); + } + }, + ) + ], + ); + } +} diff --git a/extras/drift_devtools_extension/lib/src/service.dart b/extras/drift_devtools_extension/lib/src/service.dart index b63f07c9..9bbb5c3c 100644 --- a/extras/drift_devtools_extension/lib/src/service.dart +++ b/extras/drift_devtools_extension/lib/src/service.dart @@ -5,6 +5,7 @@ import 'package:devtools_extensions/devtools_extensions.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; import 'package:rxdart/transformers.dart'; +import 'package:sqlite3/wasm.dart'; import 'package:vm_service/vm_service.dart'; final _serviceConnection = StreamController.broadcast(); @@ -48,3 +49,9 @@ final hotRestartEventProvider = return notifier; }); + +final sqliteProvider = FutureProvider((ref) async { + final sqlite = await WasmSqlite3.loadFromUrl(Uri.parse('sqlite3.wasm')); + sqlite.registerVirtualFileSystem(InMemoryFileSystem(), makeDefault: true); + return sqlite; +}); diff --git a/extras/drift_devtools_extension/pubspec.yaml b/extras/drift_devtools_extension/pubspec.yaml index c90f11ed..5b23e967 100644 --- a/extras/drift_devtools_extension/pubspec.yaml +++ b/extras/drift_devtools_extension/pubspec.yaml @@ -15,12 +15,14 @@ dependencies: devtools_app_shared: '>=0.0.5 <0.0.6' # 0.0.6 requires unstable Flutter db_viewer: ^1.0.3 rxdart: ^0.27.7 - flutter_riverpod: ^2.4.4 + flutter_riverpod: ^3.0.0-dev.0 vm_service: ^11.10.0 path: ^1.8.3 drift: ^2.12.1 logging: ^1.2.0 url_launcher: ^6.1.14 + drift_dev: ^2.13.1 + sqlite3: ^2.1.0 dev_dependencies: flutter_test: diff --git a/extras/drift_devtools_extension/web/sqlite3.wasm b/extras/drift_devtools_extension/web/sqlite3.wasm new file mode 120000 index 00000000..de4098cb --- /dev/null +++ b/extras/drift_devtools_extension/web/sqlite3.wasm @@ -0,0 +1 @@ +../../assets/sqlite3.wasm \ No newline at end of file