From 747b5be0ebe344918b038dd1a19a1ed97d315e5c Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 7 Aug 2023 12:10:34 +0200 Subject: [PATCH 1/4] Draft for wasm probe API --- drift/lib/src/web/wasm_setup.dart | 120 +++++++++++++++++- .../src/web/wasm_setup/dedicated_worker.dart | 40 +++--- drift/lib/src/web/wasm_setup/protocol.dart | 110 ++++++++++++---- drift/lib/src/web/wasm_setup/shared.dart | 19 +++ .../lib/src/web/wasm_setup/shared_worker.dart | 12 +- drift/lib/src/web/wasm_setup/types.dart | 45 ++++++- drift/lib/wasm.dart | 57 ++++++++- 7 files changed, 345 insertions(+), 58 deletions(-) diff --git a/drift/lib/src/web/wasm_setup.dart b/drift/lib/src/web/wasm_setup.dart index 42ccb35e..3b3cfacd 100644 --- a/drift/lib/src/web/wasm_setup.dart +++ b/drift/lib/src/web/wasm_setup.dart @@ -38,6 +38,124 @@ bool get supportsSharedWorkers => hasProperty(globalThis, 'SharedWorker'); /// Whether dedicated workers can be constructed in the current context. bool get supportsWorkers => hasProperty(globalThis, 'Worker'); +class WasmDatabaseOpener2 { + final Uri sqlite3WasmUri; + final Uri driftWorkerUri; + + final String? databaseName; + + final Set missingFeatures = {}; + final List availableImplementations = [ + WasmStorageImplementation.inMemory, + ]; + final Set<(DatabaseLocation, String)> existingDatabases = {}; + + MessagePort? _sharedWorker; + Worker? _dedicatedWorker; + + WasmDatabaseOpener2( + this.sqlite3WasmUri, + this.driftWorkerUri, + this.databaseName, + ); + + RequestCompatibilityCheck _createCompatibilityCheck() { + return RequestCompatibilityCheck(databaseName ?? 'driftCompatibilityCheck'); + } + + void _handleCompatibilityResult(CompatibilityResult result) { + missingFeatures.addAll(result.missingFeatures); + + final databaseName = this.databaseName; + + // Note that existingDatabases are only sent from workers shipped with drift + // 2.11 or later. Later drift versions need to be able to talk to newer + // workers though. + if (result.existingDatabases.isNotEmpty) { + existingDatabases.addAll(result.existingDatabases); + } + + if (databaseName != null) { + // If this opener has been created for WasmDatabase.open, we have a + // database name and can interpret the opfsExists and indexedDbExists + // fields we're getting from older workers accordingly. + if (result.opfsExists) { + existingDatabases.add((DatabaseLocation.opfs, databaseName)); + } + if (result.indexedDbExists) { + existingDatabases.add((DatabaseLocation.indexedDb, databaseName)); + } + } + } + + Future probe() async { + await _probeDedicated(); + } + + Future _probeDedicated() async { + if (supportsWorkers) { + final dedicatedWorker = + _dedicatedWorker = Worker(driftWorkerUri.toString()); + _createCompatibilityCheck().sendToWorker(dedicatedWorker); + + final workerMessages = StreamQueue( + _readMessages(dedicatedWorker.onMessage, dedicatedWorker.onError)); + + final status = await workerMessages.nextNoError + as DedicatedWorkerCompatibilityResult; + + _handleCompatibilityResult(status); + + if (status.supportsNestedWorkers && + status.canAccessOpfs && + status.supportsSharedArrayBuffers) { + availableImplementations.add(WasmStorageImplementation.opfsLocks); + } + + if (status.supportsIndexedDb) { + availableImplementations.add(WasmStorageImplementation.unsafeIndexedDb); + } + } else { + missingFeatures.add(MissingBrowserFeature.dedicatedWorkers); + } + } +} + +final class _ProbeResult extends WasmProbeResult { + @override + final List availableStorages; + + @override + final List<(DatabaseLocation, String)> existingDatabases; + + @override + final Set missingFeatures; + + final WasmDatabaseOpener2 opener; + + _ProbeResult( + this.availableStorages, + this.existingDatabases, + this.missingFeatures, + this.opener, + ); + + @override + Future open( + WasmStorageImplementation implementation, String name, + {FutureOr Function()? initializeDatabase}) async { + // TODO: implement open + throw UnimplementedError(); + } + + @override + Future deleteDatabase( + DatabaseLocation implementation, String name) async { + // TODO: implement deleteDatabase + throw UnimplementedError(); + } +} + class WasmDatabaseOpener { final Uri sqlite3WasmUri; final Uri driftWorkerUri; @@ -104,7 +222,7 @@ class WasmDatabaseOpener { } else if (_existsInOpfs && (availableImplementations .contains(WasmStorageImplementation.opfsShared) || - availableImplementations + availableImplementatioobjectns .contains(WasmStorageImplementation.opfsLocks))) { availableImplementations.removeWhere((element) => element != WasmStorageImplementation.opfsShared && diff --git a/drift/lib/src/web/wasm_setup/dedicated_worker.dart b/drift/lib/src/web/wasm_setup/dedicated_worker.dart index 1a8f27b0..1909d0b4 100644 --- a/drift/lib/src/web/wasm_setup/dedicated_worker.dart +++ b/drift/lib/src/web/wasm_setup/dedicated_worker.dart @@ -3,11 +3,9 @@ import 'dart:async'; import 'dart:html'; +import 'package:drift/wasm.dart'; import 'package:js/js_util.dart'; import 'package:sqlite3/wasm.dart'; -// ignore: implementation_imports -import 'package:sqlite3/src/wasm/js_interop/file_system_access.dart'; -import 'package:path/path.dart' as p; import '../../utils/synchronized.dart'; import 'protocol.dart'; @@ -48,32 +46,25 @@ class DedicatedDriftWorker { }); final existingServer = _servers.servers[dbName]; + var indexedDbExists = false, opfsExists = false; + final existingDatabases = <(DatabaseLocation, String)>[]; + + if (supportsOpfs) { + for (final database in await opfsDatabases()) { + existingDatabases.add((DatabaseLocation.opfs, database)); + + if (database == dbName) { + opfsExists = true; + } + } + } if (existingServer != null) { indexedDbExists = existingServer.storage.isIndexedDbBased; opfsExists = existingServer.storage.isOpfsBased; - } else { - if (supportsIndexedDb) { - indexedDbExists = await checkIndexedDbExists(dbName); - } - - if (supportsOpfs) { - final storage = storageManager!; - final pathSegments = p.url.split(pathForOpfs(dbName)); - - var directory = await storage.directory; - opfsExists = true; - - for (final segment in pathSegments) { - try { - directory = await directory.getDirectory(segment); - } on Object { - opfsExists = false; - break; - } - } - } + } else if (supportsIndexedDb) { + indexedDbExists = await checkIndexedDbExists(dbName); } DedicatedWorkerCompatibilityResult( @@ -84,6 +75,7 @@ class DedicatedDriftWorker { hasProperty(globalThis, 'SharedArrayBuffer'), opfsExists: opfsExists, indexedDbExists: indexedDbExists, + existingDatabases: existingDatabases, ).sendToClient(self); case ServeDriftDatabase(): _servers.serve(message); diff --git a/drift/lib/src/web/wasm_setup/protocol.dart b/drift/lib/src/web/wasm_setup/protocol.dart index a95b441b..2d0eac11 100644 --- a/drift/lib/src/web/wasm_setup/protocol.dart +++ b/drift/lib/src/web/wasm_setup/protocol.dart @@ -1,6 +1,7 @@ // ignore_for_file: public_member_api_docs import 'dart:html'; +import 'dart:js'; import 'package:js/js_util.dart'; import 'package:sqlite3/wasm.dart'; @@ -55,38 +56,65 @@ sealed class WasmInitializationMessage { } } +sealed class CompatibilityResult extends WasmInitializationMessage { + /// All existing databases. + /// + /// This list is only reported by the drift worker shipped with drift 2.11. + /// When an older worker is used, only [indexedDbExists] and [opfsExists] can + /// be used to check whether the database exists. + final List<(DatabaseLocation, String)> existingDatabases; + + final bool indexedDbExists; + final bool opfsExists; + + Iterable get missingFeatures; + + CompatibilityResult({ + required this.existingDatabases, + required this.indexedDbExists, + required this.opfsExists, + }); +} + /// A message used by the shared worker to report compatibility results. /// /// It describes the features available from the shared worker, which the tab /// can use to infer a desired storage implementation, or whether the shared /// worker should be used at all. -final class SharedWorkerCompatibilityResult extends WasmInitializationMessage { +final class SharedWorkerCompatibilityResult extends CompatibilityResult { static const type = 'SharedWorkerCompatibilityResult'; final bool canSpawnDedicatedWorkers; final bool dedicatedWorkersCanUseOpfs; final bool canUseIndexedDb; - final bool indexedDbExists; - final bool opfsExists; - SharedWorkerCompatibilityResult({ required this.canSpawnDedicatedWorkers, required this.dedicatedWorkersCanUseOpfs, required this.canUseIndexedDb, - required this.indexedDbExists, - required this.opfsExists, + required super.indexedDbExists, + required super.opfsExists, + required super.existingDatabases, }); factory SharedWorkerCompatibilityResult.fromJsPayload(Object payload) { - final data = (payload as List).cast(); + final asList = payload as List; + final asBooleans = asList.cast(); + + final List<(DatabaseLocation, String)> existingDatabases; + if (asList.length > 5) { + existingDatabases = EncodeLocations.readFromJs(asList[5] as JsArray); + } else { + existingDatabases = const []; + } return SharedWorkerCompatibilityResult( - canSpawnDedicatedWorkers: data[0], - dedicatedWorkersCanUseOpfs: data[1], - canUseIndexedDb: data[2], - indexedDbExists: data[3], - opfsExists: data[4], + canSpawnDedicatedWorkers: asBooleans[0], + dedicatedWorkersCanUseOpfs: asBooleans[1], + canUseIndexedDb: asBooleans[2], + indexedDbExists: asBooleans[3], + opfsExists: asBooleans[4], + existingDatabases: existingDatabases, ); } @@ -98,9 +126,11 @@ final class SharedWorkerCompatibilityResult extends WasmInitializationMessage { canUseIndexedDb, indexedDbExists, opfsExists, + existingDatabases.encodeToJs(), ]); } + @override Iterable get missingFeatures sync* { if (!canSpawnDedicatedWorkers) { yield MissingBrowserFeature.dedicatedWorkersInSharedWorkers; @@ -183,6 +213,10 @@ final class ServeDriftDatabase extends WasmInitializationMessage { final class RequestCompatibilityCheck extends WasmInitializationMessage { static const type = 'RequestCompatibilityCheck'; + /// The database name to check when reporting whether it exists already. + /// + /// Older versions of the drif worker only support checking a single database + /// name. On newer workers, this field is ignored. final String databaseName; RequestCompatibilityCheck(this.databaseName); @@ -197,8 +231,7 @@ final class RequestCompatibilityCheck extends WasmInitializationMessage { } } -final class DedicatedWorkerCompatibilityResult - extends WasmInitializationMessage { +final class DedicatedWorkerCompatibilityResult extends CompatibilityResult { static const type = 'DedicatedWorkerCompatibilityResult'; final bool supportsNestedWorkers; @@ -206,22 +239,24 @@ final class DedicatedWorkerCompatibilityResult final bool supportsSharedArrayBuffers; final bool supportsIndexedDb; - /// Whether an IndexedDb database under the desired name exists already. - final bool indexedDbExists; - - /// Whether an OPFS database under the desired name exists already. - final bool opfsExists; - DedicatedWorkerCompatibilityResult({ required this.supportsNestedWorkers, required this.canAccessOpfs, required this.supportsSharedArrayBuffers, required this.supportsIndexedDb, - required this.indexedDbExists, - required this.opfsExists, + required super.indexedDbExists, + required super.opfsExists, + required super.existingDatabases, }); factory DedicatedWorkerCompatibilityResult.fromJsPayload(Object payload) { + final existingDatabases = <(DatabaseLocation, String)>[]; + + if (hasProperty(payload, 'existing')) { + existingDatabases + .addAll(EncodeLocations.readFromJs(getProperty(payload, 'existing'))); + } + return DedicatedWorkerCompatibilityResult( supportsNestedWorkers: getProperty(payload, 'supportsNestedWorkers'), canAccessOpfs: getProperty(payload, 'canAccessOpfs'), @@ -230,6 +265,7 @@ final class DedicatedWorkerCompatibilityResult supportsIndexedDb: getProperty(payload, 'supportsIndexedDb'), indexedDbExists: getProperty(payload, 'indexedDbExists'), opfsExists: getProperty(payload, 'opfsExists'), + existingDatabases: existingDatabases, ); } @@ -244,10 +280,12 @@ final class DedicatedWorkerCompatibilityResult object, 'supportsSharedArrayBuffers', supportsSharedArrayBuffers); setProperty(object, 'indexedDbExists', indexedDbExists); setProperty(object, 'opfsExists', opfsExists); + setProperty(object, 'existing', existingDatabases.encodeToJs()); sender.sendTyped(type, object); } + @override Iterable get missingFeatures sync* { if (!canAccessOpfs) { yield MissingBrowserFeature.fileSystemAccess; @@ -261,6 +299,34 @@ final class DedicatedWorkerCompatibilityResult } } +extension EncodeLocations on List<(DatabaseLocation, String)> { + static List<(DatabaseLocation, String)> readFromJs(JsArray object) { + final existing = <(DatabaseLocation, String)>[]; + + for (final entry in object) { + existing.add(( + DatabaseLocation.byName[getProperty(entry as Object, 'l')]!, + getProperty(entry, 'n'), + )); + } + + return existing; + } + + Object encodeToJs() { + final existing = JsArray(); + for (final entry in this) { + final object = newObject(); + setProperty(object, 'l', entry.$1.name); + setProperty(object, 'n', entry.$2); + + existing.add(object); + } + + return existing; + } +} + final class StartFileSystemServer extends WasmInitializationMessage { static const type = 'StartFileSystemServer'; diff --git a/drift/lib/src/web/wasm_setup/shared.dart b/drift/lib/src/web/wasm_setup/shared.dart index eb1643da..106053fa 100644 --- a/drift/lib/src/web/wasm_setup/shared.dart +++ b/drift/lib/src/web/wasm_setup/shared.dart @@ -113,6 +113,25 @@ String pathForOpfs(String databaseName) { return 'drift_db/$databaseName'; } +/// Collects all drift OPFS databases. +Future> opfsDatabases() async { + final storage = storageManager; + if (storage == null) return const []; + + var directory = await storage.directory; + try { + directory = await directory.getDirectory('drift_db'); + } on Object { + // The drift_db folder doesn't exist, so there aren't any databases. + return const []; + } + + return [ + await for (final entry in directory.list()) + if (entry.isDirectory) entry.name, + ]; +} + /// Manages drift servers. /// /// When using a shared worker, multiple clients may want to use different drift diff --git a/drift/lib/src/web/wasm_setup/shared_worker.dart b/drift/lib/src/web/wasm_setup/shared_worker.dart index fee2bfdb..c125f9e7 100644 --- a/drift/lib/src/web/wasm_setup/shared_worker.dart +++ b/drift/lib/src/web/wasm_setup/shared_worker.dart @@ -76,6 +76,7 @@ class SharedDriftWorker { canUseIndexedDb: canUseIndexedDb, indexedDbExists: indexedDbExists, opfsExists: false, + existingDatabases: const [], ); } else { final worker = _dedicatedWorker ??= Worker(Uri.base.toString()); @@ -86,7 +87,12 @@ class SharedDriftWorker { final completer = Completer(); StreamSubscription? messageSubscription, errorSubscription; - void result(bool opfsAvailable, bool opfsExists, bool indexedDbExists) { + void result( + bool opfsAvailable, + bool opfsExists, + bool indexedDbExists, + List databases, + ) { if (!completer.isCompleted) { completer.complete(SharedWorkerCompatibilityResult( canSpawnDedicatedWorkers: true, @@ -94,6 +100,7 @@ class SharedDriftWorker { canUseIndexedDb: canUseIndexedDb, indexedDbExists: indexedDbExists, opfsExists: opfsExists, + existingDatabases: databases, )); messageSubscription?.cancel(); @@ -110,11 +117,12 @@ class SharedDriftWorker { compatibilityResult.canAccessOpfs, compatibilityResult.opfsExists, compatibilityResult.indexedDbExists, + compatibilityResult.existingDatabases, ); }); errorSubscription = worker.onError.listen((event) { - result(false, false, false); + result(false, false, false, const []); worker.terminate(); _dedicatedWorker = null; }); diff --git a/drift/lib/src/web/wasm_setup/types.dart b/drift/lib/src/web/wasm_setup/types.dart index d9801613..1c28b849 100644 --- a/drift/lib/src/web/wasm_setup/types.dart +++ b/drift/lib/src/web/wasm_setup/types.dart @@ -4,6 +4,8 @@ /// integration tests on a Dart VM (`extras/integration_tests/web_wasm`). library; +import 'dart:async'; + import 'package:drift/drift.dart'; /// The storage implementation used by the `drift` and `sqlite3` packages to @@ -80,6 +82,13 @@ enum WasmStorageImplementation { inMemory, } +enum DatabaseLocation { + opfs, + indexedDb; + + static final byName = DatabaseLocation.values.asNameMap(); +} + /// An enumeration of features not supported by the current browsers. /// /// While this information may not be useful to end users, it can be used to @@ -118,8 +127,40 @@ enum MissingBrowserFeature { sharedArrayBuffers, } -/// The result of opening a WASM database. -class WasmDatabaseResult { +typedef ExistingDatabase = (DatabaseLocation, String); + +abstract interface class WasmProbeResult { + /// All available [WasmStorageImplementation]s supported by the current + /// browsing context. + /// + /// Depending on the features available in the browser your app runs on and + /// whether your app is served with the required headers for shared array + /// buffers, different implementations might be available. + /// + /// You can see the [WasmStorageImplementation]s and + /// [the web documentation](https://drift.simonbinder.eu/web/#storages) to + /// learn more about which implementations drift can use. + List get availableStorages; + + /// For every storage found, drift also reports existing drift databases. + List get existingDatabases; + + /// An enumeration of missing browser features probed by drift. + /// + /// Missing browser features limit the available storage implementations. + Set get missingFeatures; + + Future open( + WasmStorageImplementation implementation, + String name, { + FutureOr Function()? initializeDatabase, + }); + + Future deleteDatabase(DatabaseLocation implementation, String name); +} + +/// The result of opening a WASM database with default options. +final class WasmDatabaseResult { /// The drift database connection to pass to the [GeneratedDatabase.new] /// constructor of your database class to use the opened database. final DatabaseConnection resolvedExecutor; diff --git a/drift/lib/wasm.dart b/drift/lib/wasm.dart index 9d962b3c..9a8f1009 100644 --- a/drift/lib/wasm.dart +++ b/drift/lib/wasm.dart @@ -10,6 +10,7 @@ import 'dart:async'; import 'dart:html'; import 'dart:typed_data'; +import 'package:collection/collection.dart'; import 'package:sqlite3/wasm.dart'; import 'backends.dart'; @@ -90,15 +91,57 @@ class WasmDatabase extends DelegatedDatabase { required Uri sqlite3Uri, required Uri driftWorkerUri, FutureOr Function()? initializeDatabase, - }) { - return WasmDatabaseOpener( - databaseName: databaseName, - sqlite3WasmUri: sqlite3Uri, - driftWorkerUri: driftWorkerUri, - initializeDatabase: initializeDatabase, - ).open(); + }) async { + final probed = + await probe(sqlite3Uri: sqlite3Uri, driftWorkerUri: driftWorkerUri); + + // If we have an existing database in storage, we want to keep using that + // format to avoid data loss (e.g. after a browser update that enables a + // otherwise preferred storage implementation). In the future, we might want + // to consider migrating between storage implementations as well. + final availableImplementations = probed.availableStorages.toList(); + + checkExisting: + for (final (location, name) in probed.existingDatabases) { + if (name == databaseName) { + final implementationsForStorage = switch (location) { + DatabaseLocation.indexedDb => const [ + WasmStorageImplementation.sharedIndexedDb, + WasmStorageImplementation.unsafeIndexedDb + ], + DatabaseLocation.opfs => const [ + WasmStorageImplementation.opfsShared, + WasmStorageImplementation.opfsLocks, + ], + }; + + // If any of the implementations for this location is still availalable, + // we want to use it instead of another location. + if (implementationsForStorage.any(availableImplementations.contains)) { + availableImplementations + .removeWhere((i) => !implementationsForStorage.contains(i)); + break checkExisting; + } + } + } + + // Enum values are ordered by preferrability, so just pick the best option + // left. + availableImplementations.sortBy((element) => element.index); + + final bestImplementation = availableImplementations.firstOrNull ?? + WasmStorageImplementation.inMemory; + final connection = await probed.open(bestImplementation, databaseName); + + return WasmDatabaseResult( + connection, bestImplementation, probed.missingFeatures); } + static Future probe({ + required Uri sqlite3Uri, + required Uri driftWorkerUri, + }) {} + /// The entrypoint for a web worker suitable for use with [open]. /// /// Generally, you can grab a pre-compiled worker file from a From b774290b3aa7a1dc8acc2b718002a9a1c497e14b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Mon, 7 Aug 2023 22:26:36 +0200 Subject: [PATCH 2/4] Fix wasm tests --- drift/lib/src/web/wasm_setup.dart | 273 +++++------------- drift/lib/src/web/wasm_setup/protocol.dart | 5 +- drift/lib/wasm.dart | 36 ++- extras/integration_tests/web_wasm/README.md | 19 +- .../integration_tests/web_wasm/web/main.dart | 113 ++++---- 5 files changed, 180 insertions(+), 266 deletions(-) diff --git a/drift/lib/src/web/wasm_setup.dart b/drift/lib/src/web/wasm_setup.dart index 3b3cfacd..b925d41b 100644 --- a/drift/lib/src/web/wasm_setup.dart +++ b/drift/lib/src/web/wasm_setup.dart @@ -13,7 +13,6 @@ import 'dart:async'; import 'dart:html'; import 'package:async/async.dart'; -import 'package:collection/collection.dart'; import 'package:drift/drift.dart'; import 'package:drift/remote.dart'; import 'package:drift/wasm.dart'; @@ -25,7 +24,6 @@ import 'package:sqlite3/wasm.dart'; import 'broadcast_stream_queries.dart'; import 'channel.dart'; import 'wasm_setup/protocol.dart'; -import 'wasm_setup/shared.dart'; /// Whether the `crossOriginIsolated` JavaScript property is true in the current /// context. @@ -38,7 +36,7 @@ bool get supportsSharedWorkers => hasProperty(globalThis, 'SharedWorker'); /// Whether dedicated workers can be constructed in the current context. bool get supportsWorkers => hasProperty(globalThis, 'Worker'); -class WasmDatabaseOpener2 { +class WasmDatabaseOpener { final Uri sqlite3WasmUri; final Uri driftWorkerUri; @@ -53,7 +51,7 @@ class WasmDatabaseOpener2 { MessagePort? _sharedWorker; Worker? _dedicatedWorker; - WasmDatabaseOpener2( + WasmDatabaseOpener( this.sqlite3WasmUri, this.driftWorkerUri, this.databaseName, @@ -89,7 +87,21 @@ class WasmDatabaseOpener2 { } Future probe() async { - await _probeDedicated(); + try { + await _probeShared(); + } on Object { + _sharedWorker?.close(); + _sharedWorker = null; + } + try { + await _probeDedicated(); + } on Object { + _dedicatedWorker?.terminate(); + _dedicatedWorker = null; + } + + return _ProbeResult(availableImplementations, existingDatabases.toList(), + missingFeatures, this); } Future _probeDedicated() async { @@ -119,9 +131,40 @@ class WasmDatabaseOpener2 { missingFeatures.add(MissingBrowserFeature.dedicatedWorkers); } } + + Future _probeShared() async { + if (supportsSharedWorkers) { + final sharedWorker = + SharedWorker(driftWorkerUri.toString(), 'drift worker'); + final port = _sharedWorker = sharedWorker.port!; + + final sharedMessages = + StreamQueue(_readMessages(port.onMessage, sharedWorker.onError)); + + // First, the shared worker will tell us which features it supports. + _createCompatibilityCheck().sendToPort(port); + final sharedFeatures = + await sharedMessages.nextNoError as SharedWorkerCompatibilityResult; + await sharedMessages.cancel(); + + _handleCompatibilityResult(sharedFeatures); + + // Prefer to use the shared worker to host the database if it supports the + // necessary APIs. + if (sharedFeatures.canSpawnDedicatedWorkers && + sharedFeatures.dedicatedWorkersCanUseOpfs) { + availableImplementations.add(WasmStorageImplementation.opfsShared); + } + if (sharedFeatures.canUseIndexedDb) { + availableImplementations.add(WasmStorageImplementation.sharedIndexedDb); + } + } else { + missingFeatures.add(MissingBrowserFeature.sharedWorkers); + } + } } -final class _ProbeResult extends WasmProbeResult { +final class _ProbeResult implements WasmProbeResult { @override final List availableStorages; @@ -131,7 +174,7 @@ final class _ProbeResult extends WasmProbeResult { @override final Set missingFeatures; - final WasmDatabaseOpener2 opener; + final WasmDatabaseOpener opener; _ProbeResult( this.availableStorages, @@ -142,124 +185,27 @@ final class _ProbeResult extends WasmProbeResult { @override Future open( - WasmStorageImplementation implementation, String name, - {FutureOr Function()? initializeDatabase}) async { - // TODO: implement open - throw UnimplementedError(); - } - - @override - Future deleteDatabase( - DatabaseLocation implementation, String name) async { - // TODO: implement deleteDatabase - throw UnimplementedError(); - } -} - -class WasmDatabaseOpener { - final Uri sqlite3WasmUri; - final Uri driftWorkerUri; - final String databaseName; - FutureOr Function()? initializeDatabase; - - final Set missingFeatures = {}; - final List availableImplementations = [ - WasmStorageImplementation.inMemory, - ]; - - bool _existsInIndexedDb = false; - bool _existsInOpfs = false; - - MessagePort? _sharedWorker; - Worker? _dedicatedWorker; - - WasmDatabaseOpener({ - required this.sqlite3WasmUri, - required this.driftWorkerUri, - required this.databaseName, - this.initializeDatabase, - }); - - Future probe() async { - try { - await _probeShared(); - } on Object { - _sharedWorker?.close(); - _sharedWorker = null; - } - try { - await _probeDedicated(); - } on Object { - _dedicatedWorker?.terminate(); - _dedicatedWorker = null; - } - - if (_dedicatedWorker == null) { - // Something is wrong with web workers, let's see if we can get things - // running without a worker. - if (await checkIndexedDbSupport()) { - availableImplementations.add(WasmStorageImplementation.unsafeIndexedDb); - _existsInIndexedDb = await checkIndexedDbExists(databaseName); - } - } - } - - Future open() async { - await probe(); - - // If we have an existing database in storage, we want to keep using that - // format to avoid data loss (e.g. after a browser update that enables a - // otherwise preferred storage implementation). In the future, we might want - // to consider migrating between storage implementations as well. - if (_existsInIndexedDb && - (availableImplementations - .contains(WasmStorageImplementation.sharedIndexedDb) || - availableImplementations - .contains(WasmStorageImplementation.unsafeIndexedDb))) { - availableImplementations.removeWhere((element) => - element != WasmStorageImplementation.sharedIndexedDb && - element != WasmStorageImplementation.unsafeIndexedDb); - } else if (_existsInOpfs && - (availableImplementations - .contains(WasmStorageImplementation.opfsShared) || - availableImplementatioobjectns - .contains(WasmStorageImplementation.opfsLocks))) { - availableImplementations.removeWhere((element) => - element != WasmStorageImplementation.opfsShared && - element != WasmStorageImplementation.opfsLocks); - } - - // Enum values are ordered by preferrability, so just pick the best option - // left. - availableImplementations.sortBy((element) => element.index); - return await _connect(availableImplementations.firstOrNull ?? - WasmStorageImplementation.inMemory); - } - - /// Opens a database with the given [storage] implementation, bypassing the - /// feature detection. Must be called after [probe]. - Future openWith(WasmStorageImplementation storage) async { - return await _connect(storage); - } - - Future _connect(WasmStorageImplementation storage) async { + WasmStorageImplementation implementation, + String name, { + FutureOr Function()? initializeDatabase, + }) async { final channel = MessageChannel(); final initializer = initializeDatabase; final initChannel = initializer != null ? MessageChannel() : null; final local = channel.port1.channel(); final message = ServeDriftDatabase( - sqlite3WasmUri: sqlite3WasmUri, + sqlite3WasmUri: opener.sqlite3WasmUri, port: channel.port2, - storage: storage, - databaseName: databaseName, + storage: implementation, + databaseName: name, initializationPort: initChannel?.port2, ); - final sharedWorker = _sharedWorker; - final dedicatedWorker = _dedicatedWorker; + final sharedWorker = opener._sharedWorker; + final dedicatedWorker = opener._dedicatedWorker; - switch (storage) { + switch (implementation) { case WasmStorageImplementation.opfsShared: case WasmStorageImplementation.sharedIndexedDb: // These are handled by the shared worker, so we can close the dedicated @@ -275,14 +221,14 @@ class WasmDatabaseOpener { } else { // Workers seem to be broken, but we don't need them with this storage // mode. - return _hostDatabaseLocally( - storage, await IndexedDbFileSystem.open(dbName: databaseName)); + return _hostDatabaseLocally(implementation, + await IndexedDbFileSystem.open(dbName: name), initializeDatabase); } - case WasmStorageImplementation.inMemory: // Nothing works on this browser, so we'll fall back to an in-memory // database. - return _hostDatabaseLocally(storage, InMemoryFileSystem()); + return _hostDatabaseLocally( + implementation, InMemoryFileSystem(), initializeDatabase); } initChannel?.port1.onMessage.listen((event) async { @@ -299,7 +245,7 @@ class WasmDatabaseOpener { }); var connection = await connectToRemoteAndInitialize(local); - if (storage == WasmStorageImplementation.opfsLocks) { + if (implementation == WasmStorageImplementation.opfsLocks) { // We want stream queries to update for writes in other tabs. For the // implementations backed by a shared worker, the worker takes care of // that. @@ -311,20 +257,21 @@ class WasmDatabaseOpener { connection = DatabaseConnection( connection.executor, connectionData: connection.connectionData, - streamQueries: BroadcastStreamQueryStore(databaseName), + streamQueries: BroadcastStreamQueryStore(name), ); } } - return WasmDatabaseResult(connection, storage, missingFeatures); + return connection; } /// Returns a database connection that doesn't use web workers. - Future _hostDatabaseLocally( - WasmStorageImplementation storage, VirtualFileSystem vfs) async { - final initializer = initializeDatabase; - - final sqlite3 = await WasmSqlite3.loadFromUrl(sqlite3WasmUri); + Future _hostDatabaseLocally( + WasmStorageImplementation storage, + VirtualFileSystem vfs, + FutureOr Function()? initializer, + ) async { + final sqlite3 = await WasmSqlite3.loadFromUrl(opener.sqlite3WasmUri); sqlite3.registerVirtualFileSystem(vfs); if (initializer != null) { @@ -338,76 +285,16 @@ class WasmDatabaseOpener { } } - return WasmDatabaseResult( - DatabaseConnection( - WasmDatabase(sqlite3: sqlite3, path: '/database'), - ), - storage, - missingFeatures, + return DatabaseConnection( + WasmDatabase(sqlite3: sqlite3, path: '/database'), ); } - Future _probeShared() async { - if (supportsSharedWorkers) { - final sharedWorker = - SharedWorker(driftWorkerUri.toString(), 'drift worker'); - final port = _sharedWorker = sharedWorker.port!; - - final sharedMessages = - StreamQueue(_readMessages(port.onMessage, sharedWorker.onError)); - - // First, the shared worker will tell us which features it supports. - RequestCompatibilityCheck(databaseName).sendToPort(port); - final sharedFeatures = - await sharedMessages.nextNoError as SharedWorkerCompatibilityResult; - await sharedMessages.cancel(); - missingFeatures.addAll(sharedFeatures.missingFeatures); - - _existsInOpfs |= sharedFeatures.opfsExists; - _existsInIndexedDb |= sharedFeatures.indexedDbExists; - - // Prefer to use the shared worker to host the database if it supports the - // necessary APIs. - if (sharedFeatures.canSpawnDedicatedWorkers && - sharedFeatures.dedicatedWorkersCanUseOpfs) { - availableImplementations.add(WasmStorageImplementation.opfsShared); - } - if (sharedFeatures.canUseIndexedDb) { - availableImplementations.add(WasmStorageImplementation.sharedIndexedDb); - } - } else { - missingFeatures.add(MissingBrowserFeature.sharedWorkers); - } - } - - Future _probeDedicated() async { - if (supportsWorkers) { - final dedicatedWorker = - _dedicatedWorker = Worker(driftWorkerUri.toString()); - RequestCompatibilityCheck(databaseName).sendToWorker(dedicatedWorker); - - final workerMessages = StreamQueue( - _readMessages(dedicatedWorker.onMessage, dedicatedWorker.onError)); - - final status = await workerMessages.nextNoError - as DedicatedWorkerCompatibilityResult; - missingFeatures.addAll(status.missingFeatures); - - _existsInOpfs |= status.opfsExists; - _existsInIndexedDb |= status.indexedDbExists; - - if (status.supportsNestedWorkers && - status.canAccessOpfs && - status.supportsSharedArrayBuffers) { - availableImplementations.add(WasmStorageImplementation.opfsLocks); - } - - if (status.supportsIndexedDb) { - availableImplementations.add(WasmStorageImplementation.unsafeIndexedDb); - } - } else { - missingFeatures.add(MissingBrowserFeature.dedicatedWorkers); - } + @override + Future deleteDatabase( + DatabaseLocation implementation, String name) async { + // TODO: implement deleteDatabase + throw UnimplementedError(); } } diff --git a/drift/lib/src/web/wasm_setup/protocol.dart b/drift/lib/src/web/wasm_setup/protocol.dart index 2d0eac11..a663004d 100644 --- a/drift/lib/src/web/wasm_setup/protocol.dart +++ b/drift/lib/src/web/wasm_setup/protocol.dart @@ -103,7 +103,8 @@ final class SharedWorkerCompatibilityResult extends CompatibilityResult { final List<(DatabaseLocation, String)> existingDatabases; if (asList.length > 5) { - existingDatabases = EncodeLocations.readFromJs(asList[5] as JsArray); + existingDatabases = + EncodeLocations.readFromJs(asList[5] as List); } else { existingDatabases = const []; } @@ -300,7 +301,7 @@ final class DedicatedWorkerCompatibilityResult extends CompatibilityResult { } extension EncodeLocations on List<(DatabaseLocation, String)> { - static List<(DatabaseLocation, String)> readFromJs(JsArray object) { + static List<(DatabaseLocation, String)> readFromJs(List object) { final existing = <(DatabaseLocation, String)>[]; for (final entry in object) { diff --git a/drift/lib/wasm.dart b/drift/lib/wasm.dart index 9a8f1009..6b4cc9b4 100644 --- a/drift/lib/wasm.dart +++ b/drift/lib/wasm.dart @@ -11,11 +11,11 @@ import 'dart:html'; import 'dart:typed_data'; import 'package:collection/collection.dart'; +import 'package:drift/src/web/wasm_setup.dart'; import 'package:sqlite3/wasm.dart'; import 'backends.dart'; import 'src/sqlite3/database.dart'; -import 'src/web/wasm_setup.dart'; import 'src/web/wasm_setup/dedicated_worker.dart'; import 'src/web/wasm_setup/shared_worker.dart'; import 'src/web/wasm_setup/types.dart'; @@ -92,8 +92,11 @@ class WasmDatabase extends DelegatedDatabase { required Uri driftWorkerUri, FutureOr Function()? initializeDatabase, }) async { - final probed = - await probe(sqlite3Uri: sqlite3Uri, driftWorkerUri: driftWorkerUri); + final probed = await probe( + sqlite3Uri: sqlite3Uri, + driftWorkerUri: driftWorkerUri, + databaseName: databaseName, + ); // If we have an existing database in storage, we want to keep using that // format to avoid data loss (e.g. after a browser update that enables a @@ -137,10 +140,35 @@ class WasmDatabase extends DelegatedDatabase { connection, bestImplementation, probed.missingFeatures); } + /// Probes for: + /// + /// - available storage implementations based on supported web APIs. + /// - APIs not currently supported by the browser. + /// - existing drift databases in the current browsing context. + /// + /// This information can be used to control whether to open a drift database, + /// or whether the current browser is unsuitable for the persistence + /// requirements of your app. + /// For most apps, using [open] directly is easier. It calls [probe] + /// internally and uses the best storage implementation available. + /// + /// The [databaseName] option is not strictly required. But drift can't list + /// databases stored in IndexedDb, they are not part of + /// [WasmProbeResult.existingDatabases] by default. This is because drift + /// databases can't be distinguished from other IndexedDb databases without + /// opening them, which might disturb the running operation of them. When a + /// [databaseName] is passed, drift will explicitly probe whether a database + /// with that name exists in IndexedDb and whether it is a drift database. + /// Drift is always able to list databases stored in OPFS, regardless of + /// whether [databaseName] is passed or not. static Future probe({ required Uri sqlite3Uri, required Uri driftWorkerUri, - }) {} + String? databaseName, + }) async { + return await WasmDatabaseOpener(sqlite3Uri, driftWorkerUri, databaseName) + .probe(); + } /// The entrypoint for a web worker suitable for use with [open]. /// diff --git a/extras/integration_tests/web_wasm/README.md b/extras/integration_tests/web_wasm/README.md index 00479a29..f3b62bd1 100644 --- a/extras/integration_tests/web_wasm/README.md +++ b/extras/integration_tests/web_wasm/README.md @@ -1,18 +1,7 @@ Integration tests for `package:drift/native.dart`. -To test persistence, we need to instrument a browsers in a way not covered by the normal -`test` package. For instance, we need to reload pages to ensure data is still there. +To run the tests automatically (with us managing a browser driver), just run `dart test`. -## Running tests with Firefox - -``` -geckodriver & -dart run tool/drift_wasm_test.dart firefox http://localhost:4444 -``` - -## Running tests with Chrome - -``` -chromedriver --port=4444 --url-base=wd/hub & -dart run tool/drift_wasm_test.dart chrome http://localhost:4444/wd/hub/ -``` +To manually debug issues, it might make sense to trigger some functionality manually. +You can run `dart run tool/server_manually.dart` to start a web server hosting the test +content on http://localhost:8080. \ No newline at end of file diff --git a/extras/integration_tests/web_wasm/web/main.dart b/extras/integration_tests/web_wasm/web/main.dart index 8eabb8ab..3773f047 100644 --- a/extras/integration_tests/web_wasm/web/main.dart +++ b/extras/integration_tests/web_wasm/web/main.dart @@ -5,14 +5,15 @@ import 'dart:js_util'; import 'package:async/async.dart'; import 'package:drift/drift.dart'; import 'package:drift/wasm.dart'; -// ignore: invalid_use_of_internal_member -import 'package:drift/src/web/wasm_setup.dart'; import 'package:http/http.dart' as http; import 'package:web_wasm/initialization_mode.dart'; import 'package:web_wasm/src/database.dart'; import 'package:sqlite3/wasm.dart'; const dbName = 'drift_test'; +final sqlite3WasmUri = Uri.parse('/sqlite3.wasm'); +final driftWorkerUri = Uri.parse('/worker.dart.js'); + TestDatabase? openedDatabase; StreamQueue? tableUpdates; @@ -31,7 +32,12 @@ void main() { document.getElementById('selfcheck')?.onClick.listen((event) async { print('starting'); - final database = await _opener.open(); + final database = await WasmDatabase.open( + databaseName: dbName, + sqlite3Uri: sqlite3WasmUri, + driftWorkerUri: driftWorkerUri, + initializeDatabase: _initializeDatabase, + ); print('selected storage: ${database.chosenImplementation}'); print('missing features: ${database.missingFeatures}'); @@ -54,77 +60,80 @@ void _addCallbackForWebDriver(String name, Future Function(String?) impl) { })); } -WasmDatabaseOpener get _opener { - Future Function()? initializeDatabase; - +Future _initializeDatabase() async { switch (initializationMode) { case InitializationMode.loadAsset: - initializeDatabase = () async { - final response = await http.get(Uri.parse('/initial.db')); - return response.bodyBytes; - }; + final response = await http.get(Uri.parse('/initial.db')); + return response.bodyBytes; + case InitializationMode.migrateCustomWasmDatabase: - initializeDatabase = () async { - // Let's first open a custom WasmDatabase, the way it would have been - // done before WasmDatabase.open. - final sqlite3 = - await WasmSqlite3.loadFromUrl(Uri.parse('/sqlite3.wasm')); - final fs = await IndexedDbFileSystem.open(dbName: dbName); - sqlite3.registerVirtualFileSystem(fs, makeDefault: true); - final wasmDb = WasmDatabase(sqlite3: sqlite3, path: '/app.db'); - final db = TestDatabase(wasmDb); - await db - .into(db.testTable) - .insert(TestTableCompanion.insert(content: 'from old database')); - await db.close(); + // Let's first open a custom WasmDatabase, the way it would have been + // done before WasmDatabase.open. + final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('/sqlite3.wasm')); + final fs = await IndexedDbFileSystem.open(dbName: dbName); + sqlite3.registerVirtualFileSystem(fs, makeDefault: true); - final (file: file, outFlags: _) = - fs.xOpen(Sqlite3Filename('/app.db'), 0); - final blob = Uint8List(file.xFileSize()); - file.xRead(blob, 0); - file.xClose(); - fs.xDelete('/app.db', 0); - await fs.close(); + final wasmDb = WasmDatabase(sqlite3: sqlite3, path: '/app.db'); + final db = TestDatabase(wasmDb); + await db + .into(db.testTable) + .insert(TestTableCompanion.insert(content: 'from old database')); + await db.close(); - return blob; - }; - break; + final (file: file, outFlags: _) = fs.xOpen(Sqlite3Filename('/app.db'), 0); + final blob = Uint8List(file.xFileSize()); + file.xRead(blob, 0); + file.xClose(); + fs.xDelete('/app.db', 0); + await fs.close(); + + return blob; case InitializationMode.none: - break; + return null; } - - return WasmDatabaseOpener( - databaseName: dbName, - sqlite3WasmUri: Uri.parse('/sqlite3.wasm'), - driftWorkerUri: Uri.parse('/worker.dart.js'), - initializeDatabase: initializeDatabase, - ); } Future _detectImplementations(String? _) async { - final opener = _opener; - await opener.probe(); + final result = await WasmDatabase.probe( + sqlite3Uri: sqlite3WasmUri, + driftWorkerUri: driftWorkerUri, + databaseName: dbName, + ); return json.encode({ - 'impls': opener.availableImplementations.map((r) => r.name).toList(), - 'missing': opener.missingFeatures.map((r) => r.name).toList(), + 'impls': result.availableStorages.map((r) => r.name).toList(), + 'missing': result.missingFeatures.map((r) => r.name).toList(), }); } Future _open(String? implementationName) async { - final opener = _opener; - WasmDatabaseResult result; + DatabaseConnection connection; if (implementationName != null) { - await opener.probe(); - result = await opener - .openWith(WasmStorageImplementation.values.byName(implementationName)); + final probeResult = await WasmDatabase.probe( + sqlite3Uri: sqlite3WasmUri, + driftWorkerUri: driftWorkerUri, + databaseName: dbName, + ); + + connection = await probeResult.open( + WasmStorageImplementation.values.byName(implementationName), + dbName, + initializeDatabase: _initializeDatabase, + ); } else { - result = await opener.open(); + final result = await WasmDatabase.open( + databaseName: dbName, + sqlite3Uri: sqlite3WasmUri, + driftWorkerUri: driftWorkerUri, + initializeDatabase: _initializeDatabase, + ); + + connection = result.resolvedExecutor; } - final db = openedDatabase = TestDatabase(result.resolvedExecutor); + final db = openedDatabase = TestDatabase(connection); // Make sure it works! await db.customSelect('SELECT 1').get(); From 9d4d3cd2a4a884fda186f9183c6ca49e6374adcf Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Aug 2023 18:13:45 +0200 Subject: [PATCH 3/4] Support deleting databases --- drift/lib/src/web/wasm_setup.dart | 103 ++++++++++++------ .../src/web/wasm_setup/dedicated_worker.dart | 20 +++- drift/lib/src/web/wasm_setup/protocol.dart | 70 ++++++++---- drift/lib/src/web/wasm_setup/shared.dart | 32 +++++- drift/lib/src/web/wasm_setup/types.dart | 34 +++++- drift/lib/wasm.dart | 4 +- .../web_wasm/lib/driver.dart | 14 +++ .../web_wasm/test/drift_wasm_test.dart | 23 ++++ .../integration_tests/web_wasm/web/main.dart | 13 +++ 9 files changed, 246 insertions(+), 67 deletions(-) diff --git a/drift/lib/src/web/wasm_setup.dart b/drift/lib/src/web/wasm_setup.dart index b925d41b..3c7a0697 100644 --- a/drift/lib/src/web/wasm_setup.dart +++ b/drift/lib/src/web/wasm_setup.dart @@ -23,6 +23,7 @@ import 'package:sqlite3/wasm.dart'; import 'broadcast_stream_queries.dart'; import 'channel.dart'; +import 'wasm_setup/shared.dart'; import 'wasm_setup/protocol.dart'; /// Whether the `crossOriginIsolated` JavaScript property is true in the current @@ -46,10 +47,9 @@ class WasmDatabaseOpener { final List availableImplementations = [ WasmStorageImplementation.inMemory, ]; - final Set<(DatabaseLocation, String)> existingDatabases = {}; + final Set existingDatabases = {}; - MessagePort? _sharedWorker; - Worker? _dedicatedWorker; + _DriftWorker? _sharedWorker, _dedicatedWorker; WasmDatabaseOpener( this.sqlite3WasmUri, @@ -78,10 +78,10 @@ class WasmDatabaseOpener { // database name and can interpret the opfsExists and indexedDbExists // fields we're getting from older workers accordingly. if (result.opfsExists) { - existingDatabases.add((DatabaseLocation.opfs, databaseName)); + existingDatabases.add((WebStorageApi.opfs, databaseName)); } if (result.indexedDbExists) { - existingDatabases.add((DatabaseLocation.indexedDb, databaseName)); + existingDatabases.add((WebStorageApi.indexedDb, databaseName)); } } } @@ -96,7 +96,7 @@ class WasmDatabaseOpener { try { await _probeDedicated(); } on Object { - _dedicatedWorker?.terminate(); + _dedicatedWorker?.close(); _dedicatedWorker = null; } @@ -106,14 +106,11 @@ class WasmDatabaseOpener { Future _probeDedicated() async { if (supportsWorkers) { - final dedicatedWorker = - _dedicatedWorker = Worker(driftWorkerUri.toString()); - _createCompatibilityCheck().sendToWorker(dedicatedWorker); + final dedicatedWorker = _dedicatedWorker = + _DriftWorker.dedicated(Worker(driftWorkerUri.toString())); + _createCompatibilityCheck().sendTo(dedicatedWorker.send); - final workerMessages = StreamQueue( - _readMessages(dedicatedWorker.onMessage, dedicatedWorker.onError)); - - final status = await workerMessages.nextNoError + final status = await dedicatedWorker.workerMessages.nextNoError as DedicatedWorkerCompatibilityResult; _handleCompatibilityResult(status); @@ -136,16 +133,13 @@ class WasmDatabaseOpener { if (supportsSharedWorkers) { final sharedWorker = SharedWorker(driftWorkerUri.toString(), 'drift worker'); - final port = _sharedWorker = sharedWorker.port!; - - final sharedMessages = - StreamQueue(_readMessages(port.onMessage, sharedWorker.onError)); + final port = sharedWorker.port!; + final shared = _sharedWorker = _DriftWorker.shared(sharedWorker, port); // First, the shared worker will tell us which features it supports. _createCompatibilityCheck().sendToPort(port); - final sharedFeatures = - await sharedMessages.nextNoError as SharedWorkerCompatibilityResult; - await sharedMessages.cancel(); + final sharedFeatures = await shared.workerMessages.nextNoError + as SharedWorkerCompatibilityResult; _handleCompatibilityResult(sharedFeatures); @@ -164,12 +158,50 @@ class WasmDatabaseOpener { } } +final class _DriftWorker { + final AbstractWorker worker; + + /// The message port to communicate with the worker, if it's a shared worker. + final MessagePort? portForShared; + + final StreamQueue workerMessages; + + _DriftWorker.dedicated(Worker this.worker) + : portForShared = null, + workerMessages = + StreamQueue(_readMessages(worker.onMessage, worker.onError)); + + _DriftWorker.shared(SharedWorker this.worker, this.portForShared) + : workerMessages = + StreamQueue(_readMessages(worker.port!.onMessage, worker.onError)); + + void send(Object? msg, [List? transfer]) { + switch (worker) { + case final Worker worker: + worker.postMessage(msg, transfer); + case SharedWorker(): + portForShared!.postMessage(msg, transfer); + } + } + + void close() { + workerMessages.cancel(); + + switch (worker) { + case final Worker dedicated: + dedicated.terminate(); + case SharedWorker(): + portForShared!.close(); + } + } +} + final class _ProbeResult implements WasmProbeResult { @override final List availableStorages; @override - final List<(DatabaseLocation, String)> existingDatabases; + final List existingDatabases; @override final Set missingFeatures; @@ -208,16 +240,12 @@ final class _ProbeResult implements WasmProbeResult { switch (implementation) { case WasmStorageImplementation.opfsShared: case WasmStorageImplementation.sharedIndexedDb: - // These are handled by the shared worker, so we can close the dedicated - // worker used for feature detection. - dedicatedWorker?.terminate(); - message.sendToPort(sharedWorker!); + // Forward connection request to shared worker. + message.sendTo(sharedWorker!.send); case WasmStorageImplementation.opfsLocks: case WasmStorageImplementation.unsafeIndexedDb: - sharedWorker?.close(); - if (dedicatedWorker != null) { - message.sendToWorker(dedicatedWorker); + message.sendTo(dedicatedWorker.send); } else { // Workers seem to be broken, but we don't need them with this storage // mode. @@ -291,10 +319,21 @@ final class _ProbeResult implements WasmProbeResult { } @override - Future deleteDatabase( - DatabaseLocation implementation, String name) async { - // TODO: implement deleteDatabase - throw UnimplementedError(); + Future deleteDatabase(ExistingDatabase database) async { + switch (database.$1) { + case WebStorageApi.indexedDb: + await deleteDatabaseInIndexedDb(database.$2); + case WebStorageApi.opfs: + final dedicated = opener._dedicatedWorker; + if (dedicated != null) { + DeleteDatabase(database).sendTo(dedicated.send); + + await dedicated.workerMessages.nextNoError; + } else { + throw StateError( + 'No dedicated worker available to delete OPFS database'); + } + } } } diff --git a/drift/lib/src/web/wasm_setup/dedicated_worker.dart b/drift/lib/src/web/wasm_setup/dedicated_worker.dart index 1909d0b4..30bb216c 100644 --- a/drift/lib/src/web/wasm_setup/dedicated_worker.dart +++ b/drift/lib/src/web/wasm_setup/dedicated_worker.dart @@ -48,11 +48,11 @@ class DedicatedDriftWorker { final existingServer = _servers.servers[dbName]; var indexedDbExists = false, opfsExists = false; - final existingDatabases = <(DatabaseLocation, String)>[]; + final existingDatabases = []; if (supportsOpfs) { for (final database in await opfsDatabases()) { - existingDatabases.add((DatabaseLocation.opfs, database)); + existingDatabases.add((WebStorageApi.opfs, database)); if (database == dbName) { opfsExists = true; @@ -83,6 +83,22 @@ class DedicatedDriftWorker { final worker = await VfsWorker.create(options); self.postMessage(true); await worker.start(); + case DeleteDatabase(database: (final storage, final name)): + try { + switch (storage) { + case WebStorageApi.indexedDb: + await deleteDatabaseInIndexedDb(name); + case WebStorageApi.opfs: + await deleteDatabaseInOpfs(name); + } + + // Send the request back to indicate a successful delete. + message.sendToClient(self); + } catch (e) { + WorkerError(e.toString()).sendToClient(self); + } + + break; default: break; } diff --git a/drift/lib/src/web/wasm_setup/protocol.dart b/drift/lib/src/web/wasm_setup/protocol.dart index a663004d..9c5c4150 100644 --- a/drift/lib/src/web/wasm_setup/protocol.dart +++ b/drift/lib/src/web/wasm_setup/protocol.dart @@ -30,6 +30,7 @@ sealed class WasmInitializationMessage { DedicatedWorkerCompatibilityResult.fromJsPayload(payload!), SharedWorkerCompatibilityResult.type => SharedWorkerCompatibilityResult.fromJsPayload(payload!), + DeleteDatabase.type => DeleteDatabase.fromJsPayload(payload!), _ => throw ArgumentError('Unknown type $type'), }; } @@ -62,7 +63,7 @@ sealed class CompatibilityResult extends WasmInitializationMessage { /// This list is only reported by the drift worker shipped with drift 2.11. /// When an older worker is used, only [indexedDbExists] and [opfsExists] can /// be used to check whether the database exists. - final List<(DatabaseLocation, String)> existingDatabases; + final List existingDatabases; final bool indexedDbExists; final bool opfsExists; @@ -101,7 +102,7 @@ final class SharedWorkerCompatibilityResult extends CompatibilityResult { final asList = payload as List; final asBooleans = asList.cast(); - final List<(DatabaseLocation, String)> existingDatabases; + final List existingDatabases; if (asList.length > 5) { existingDatabases = EncodeLocations.readFromJs(asList[5] as List); @@ -251,7 +252,7 @@ final class DedicatedWorkerCompatibilityResult extends CompatibilityResult { }); factory DedicatedWorkerCompatibilityResult.fromJsPayload(Object payload) { - final existingDatabases = <(DatabaseLocation, String)>[]; + final existingDatabases = []; if (hasProperty(payload, 'existing')) { existingDatabases @@ -300,13 +301,51 @@ final class DedicatedWorkerCompatibilityResult extends CompatibilityResult { } } -extension EncodeLocations on List<(DatabaseLocation, String)> { - static List<(DatabaseLocation, String)> readFromJs(List object) { - final existing = <(DatabaseLocation, String)>[]; +final class StartFileSystemServer extends WasmInitializationMessage { + static const type = 'StartFileSystemServer'; + + final WorkerOptions sqlite3Options; + + StartFileSystemServer(this.sqlite3Options); + + factory StartFileSystemServer.fromJsPayload(Object payload) { + return StartFileSystemServer(payload as WorkerOptions); + } + + @override + void sendTo(PostMessage sender) { + sender.sendTyped(type, sqlite3Options); + } +} + +final class DeleteDatabase extends WasmInitializationMessage { + static const type = 'DeleteDatabase'; + + final ExistingDatabase database; + + DeleteDatabase(this.database); + + factory DeleteDatabase.fromJsPayload(Object payload) { + final asList = payload as List; + return DeleteDatabase(( + WebStorageApi.byName[asList[0] as String]!, + asList[1] as String, + )); + } + + @override + void sendTo(PostMessage sender) { + sender.sendTyped(type, [database.$1.name, database.$2]); + } +} + +extension EncodeLocations on List { + static List readFromJs(List object) { + final existing = []; for (final entry in object) { existing.add(( - DatabaseLocation.byName[getProperty(entry as Object, 'l')]!, + WebStorageApi.byName[getProperty(entry as Object, 'l')]!, getProperty(entry, 'n'), )); } @@ -328,23 +367,6 @@ extension EncodeLocations on List<(DatabaseLocation, String)> { } } -final class StartFileSystemServer extends WasmInitializationMessage { - static const type = 'StartFileSystemServer'; - - final WorkerOptions sqlite3Options; - - StartFileSystemServer(this.sqlite3Options); - - factory StartFileSystemServer.fromJsPayload(Object payload) { - return StartFileSystemServer(payload as WorkerOptions); - } - - @override - void sendTo(PostMessage sender) { - sender.sendTyped(type, sqlite3Options); - } -} - extension on PostMessage { void sendTyped(String type, Object? payload, [List? transfer]) { final object = newObject(); diff --git a/drift/lib/src/web/wasm_setup/shared.dart b/drift/lib/src/web/wasm_setup/shared.dart index 106053fa..48623e07 100644 --- a/drift/lib/src/web/wasm_setup/shared.dart +++ b/drift/lib/src/web/wasm_setup/shared.dart @@ -87,7 +87,7 @@ Future checkIndexedDbExists(String databaseName) async { try { final idb = getProperty(globalThis, 'indexedDB'); - await idb.open( + final database = await idb.open( databaseName, // Current schema version used by the [IndexedDbFileSystem] version: 1, @@ -100,6 +100,7 @@ Future checkIndexedDbExists(String databaseName) async { ); indexedDbExists ??= true; + database.close(); } catch (_) { // May throw due to us aborting in the upgrade callback. } @@ -107,6 +108,14 @@ Future checkIndexedDbExists(String databaseName) async { return indexedDbExists ?? false; } +/// Deletes a database from IndexedDb if supported. +Future deleteDatabaseInIndexedDb(String databaseName) async { + final idb = window.indexedDB; + if (idb != null) { + await idb.deleteDatabase(databaseName); + } +} + /// Constructs the path used by drift to store a database in the origin-private /// section of the agent's file system. String pathForOpfs(String databaseName) { @@ -132,6 +141,22 @@ Future> opfsDatabases() async { ]; } +/// Deletes the OPFS folder storing a database with the given [databaseName] if +/// such folder exists. +Future deleteDatabaseInOpfs(String databaseName) async { + final storage = storageManager; + if (storage == null) return; + + var directory = await storage.directory; + try { + directory = await directory.getDirectory('drift_db'); + await directory.removeEntry(databaseName, recursive: true); + } on Object { + // fine, an error probably means that the database didn't exist in the first + // place. + } +} + /// Manages drift servers. /// /// When using a shared worker, multiple clients may want to use different drift @@ -151,7 +176,8 @@ class DriftServerController { final vfs = await switch (message.storage) { WasmStorageImplementation.opfsShared => - SimpleOpfsFileSystem.loadFromStorage(message.databaseName), + SimpleOpfsFileSystem.loadFromStorage( + pathForOpfs(message.databaseName)), WasmStorageImplementation.opfsLocks => _loadLockedWasmVfs(message.databaseName), WasmStorageImplementation.unsafeIndexedDb || @@ -190,7 +216,7 @@ class DriftServerController { Future _loadLockedWasmVfs(String databaseName) async { // Create SharedArrayBuffers to synchronize requests final options = WasmVfs.createOptions( - root: 'drift_db/$databaseName/', + root: pathForOpfs(databaseName), ); final worker = Worker(Uri.base.toString()); diff --git a/drift/lib/src/web/wasm_setup/types.dart b/drift/lib/src/web/wasm_setup/types.dart index 1c28b849..a95e2c3c 100644 --- a/drift/lib/src/web/wasm_setup/types.dart +++ b/drift/lib/src/web/wasm_setup/types.dart @@ -82,11 +82,17 @@ enum WasmStorageImplementation { inMemory, } -enum DatabaseLocation { +/// The storage API used by drift to store a database. +enum WebStorageApi { + /// The database is stored in the origin-private section of the user's file + /// system via the FileSystem Access API. opfs, + + /// The database is stored in IndexedDb. indexedDb; - static final byName = DatabaseLocation.values.asNameMap(); + /// Cached [EnumByName.asNameMap] for [values]. + static final byName = WebStorageApi.values.asNameMap(); } /// An enumeration of features not supported by the current browsers. @@ -127,8 +133,20 @@ enum MissingBrowserFeature { sharedArrayBuffers, } -typedef ExistingDatabase = (DatabaseLocation, String); +/// Information about an existing web database, consisting of its +/// storage API ([WebStorageApi]) and its name. +typedef ExistingDatabase = (WebStorageApi, String); +/// The result of probing the current browser for wasm compatibility. +/// +/// This reports available storage implementations ([availableStorages]) and +/// [missingFeatures] that contributed to some storage implementations not being +/// available. +/// +/// In addition, [existingDatabases] reports a list of existing databases. Note +/// that databases stored in IndexedDb can't be listed reliably. Only databases +/// with the name given in [WasmDatabase.probe] are listed. Databases stored in +/// OPFS are always listed. abstract interface class WasmProbeResult { /// All available [WasmStorageImplementation]s supported by the current /// browsing context. @@ -150,13 +168,21 @@ abstract interface class WasmProbeResult { /// Missing browser features limit the available storage implementations. Set get missingFeatures; + /// Opens a connection to a database via the chosen [implementation]. + /// + /// When this database doesn't exist, [initializeDatabase] is invoked to + /// optionally return the initial bytes of the database. Future open( WasmStorageImplementation implementation, String name, { FutureOr Function()? initializeDatabase, }); - Future deleteDatabase(DatabaseLocation implementation, String name); + /// Deletes an [ExistingDatabase] from storage. + /// + /// This method should not be called while a connection to the database is + /// opened. + Future deleteDatabase(ExistingDatabase database); } /// The result of opening a WASM database with default options. diff --git a/drift/lib/wasm.dart b/drift/lib/wasm.dart index 6b4cc9b4..49cf7b10 100644 --- a/drift/lib/wasm.dart +++ b/drift/lib/wasm.dart @@ -108,11 +108,11 @@ class WasmDatabase extends DelegatedDatabase { for (final (location, name) in probed.existingDatabases) { if (name == databaseName) { final implementationsForStorage = switch (location) { - DatabaseLocation.indexedDb => const [ + WebStorageApi.indexedDb => const [ WasmStorageImplementation.sharedIndexedDb, WasmStorageImplementation.unsafeIndexedDb ], - DatabaseLocation.opfs => const [ + WebStorageApi.opfs => const [ WasmStorageImplementation.opfsShared, WasmStorageImplementation.opfsLocks, ], diff --git a/extras/integration_tests/web_wasm/lib/driver.dart b/extras/integration_tests/web_wasm/lib/driver.dart index 94a13a34..7f179373 100644 --- a/extras/integration_tests/web_wasm/lib/driver.dart +++ b/extras/integration_tests/web_wasm/lib/driver.dart @@ -105,6 +105,7 @@ class DriftWebDriver { ({ Set storages, Set missingFeatures, + List existing, })> probeImplementations() async { final rawResult = await driver .executeAsync('detectImplementations("", arguments[0])', []); @@ -119,6 +120,13 @@ class DriftWebDriver { for (final entry in result['missing']) MissingBrowserFeature.values.byName(entry) }, + existing: [ + for (final entry in result['existing']) + ( + WebStorageApi.byName[entry[0] as String]!, + entry[1] as String, + ), + ], ); } @@ -149,4 +157,10 @@ class DriftWebDriver { throw 'Could not set initialization mode'; } } + + Future deleteDatabase(WebStorageApi storageApi, String name) async { + await driver.executeAsync('delete_database(arguments[0], arguments[1])', [ + json.encode([storageApi.name, name]), + ]); + } } diff --git a/extras/integration_tests/web_wasm/test/drift_wasm_test.dart b/extras/integration_tests/web_wasm/test/drift_wasm_test.dart index 2139d94e..aadfab57 100644 --- a/extras/integration_tests/web_wasm/test/drift_wasm_test.dart +++ b/extras/integration_tests/web_wasm/test/drift_wasm_test.dart @@ -118,6 +118,29 @@ void main() { } }); + if (entry != WasmStorageImplementation.inMemory) { + test('delete', () async { + final impl = await driver.probeImplementations(); + expect(impl.existing, isEmpty); + + await driver.openDatabase(entry); + await driver.insertIntoDatabase(); + await driver.waitForTableUpdate(); + + await driver.driver.refresh(); // Reset JS state + + final newImpls = await driver.probeImplementations(); + expect(newImpls.existing, hasLength(1)); + final existing = newImpls.existing[0]; + await driver.deleteDatabase(existing.$1, existing.$2); + + await driver.driver.refresh(); + + final finalImpls = await driver.probeImplementations(); + expect(finalImpls.existing, isEmpty); + }); + } + group( 'initialization from ', () { diff --git a/extras/integration_tests/web_wasm/web/main.dart b/extras/integration_tests/web_wasm/web/main.dart index 3773f047..8bb6dc54 100644 --- a/extras/integration_tests/web_wasm/web/main.dart +++ b/extras/integration_tests/web_wasm/web/main.dart @@ -29,6 +29,18 @@ void main() { initializationMode = InitializationMode.values.byName(arg!); return true; }); + _addCallbackForWebDriver('delete_database', (arg) async { + final result = await WasmDatabase.probe( + sqlite3Uri: sqlite3WasmUri, + driftWorkerUri: driftWorkerUri, + ); + + final decoded = json.decode(arg!); + + await result.deleteDatabase( + (WebStorageApi.byName[decoded[0] as String]!, decoded[1] as String), + ); + }); document.getElementById('selfcheck')?.onClick.listen((event) async { print('starting'); @@ -104,6 +116,7 @@ Future _detectImplementations(String? _) async { return json.encode({ 'impls': result.availableStorages.map((r) => r.name).toList(), 'missing': result.missingFeatures.map((r) => r.name).toList(), + 'existing': result.existingDatabases.map((r) => [r.$1.name, r.$2]).toList(), }); } From 1036d7f68536c3f15056f849abb3ccf051e47d73 Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Wed, 9 Aug 2023 18:17:25 +0200 Subject: [PATCH 4/4] Add warning about old worker files --- drift/lib/src/web/wasm_setup/types.dart | 4 ++++ drift/lib/wasm.dart | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/drift/lib/src/web/wasm_setup/types.dart b/drift/lib/src/web/wasm_setup/types.dart index a95e2c3c..86cdd182 100644 --- a/drift/lib/src/web/wasm_setup/types.dart +++ b/drift/lib/src/web/wasm_setup/types.dart @@ -182,6 +182,10 @@ abstract interface class WasmProbeResult { /// /// This method should not be called while a connection to the database is /// opened. + /// + /// This method is only supported when using the drift worker shipped with the + /// drift 2.11 release or later. This method will not work when using an older + /// worker. Future deleteDatabase(ExistingDatabase database); } diff --git a/drift/lib/wasm.dart b/drift/lib/wasm.dart index 49cf7b10..28ff49aa 100644 --- a/drift/lib/wasm.dart +++ b/drift/lib/wasm.dart @@ -161,6 +161,10 @@ class WasmDatabase extends DelegatedDatabase { /// with that name exists in IndexedDb and whether it is a drift database. /// Drift is always able to list databases stored in OPFS, regardless of /// whether [databaseName] is passed or not. + /// + /// Note that this method is only fully supported when using the drift worker + /// shipped with the drift 2.11 release. Older workers are only supported when + /// [databaseName] is non-null. static Future probe({ required Uri sqlite3Uri, required Uri driftWorkerUri,