Fix wasm tests

This commit is contained in:
Simon Binder 2023-08-07 22:26:36 +02:00
parent 747b5be0eb
commit b774290b3a
No known key found for this signature in database
GPG Key ID: 7891917E4147B8C0
5 changed files with 180 additions and 266 deletions

View File

@ -13,7 +13,6 @@ import 'dart:async';
import 'dart:html';
import 'package:async/async.dart';
import 'package:collection/collection.dart';
import 'package:drift/drift.dart';
import 'package:drift/remote.dart';
import 'package:drift/wasm.dart';
@ -25,7 +24,6 @@ import 'package:sqlite3/wasm.dart';
import 'broadcast_stream_queries.dart';
import 'channel.dart';
import 'wasm_setup/protocol.dart';
import 'wasm_setup/shared.dart';
/// Whether the `crossOriginIsolated` JavaScript property is true in the current
/// context.
@ -38,7 +36,7 @@ bool get supportsSharedWorkers => hasProperty(globalThis, 'SharedWorker');
/// Whether dedicated workers can be constructed in the current context.
bool get supportsWorkers => hasProperty(globalThis, 'Worker');
class WasmDatabaseOpener2 {
class WasmDatabaseOpener {
final Uri sqlite3WasmUri;
final Uri driftWorkerUri;
@ -53,7 +51,7 @@ class WasmDatabaseOpener2 {
MessagePort? _sharedWorker;
Worker? _dedicatedWorker;
WasmDatabaseOpener2(
WasmDatabaseOpener(
this.sqlite3WasmUri,
this.driftWorkerUri,
this.databaseName,
@ -89,7 +87,21 @@ class WasmDatabaseOpener2 {
}
Future<WasmProbeResult> probe() async {
await _probeDedicated();
try {
await _probeShared();
} on Object {
_sharedWorker?.close();
_sharedWorker = null;
}
try {
await _probeDedicated();
} on Object {
_dedicatedWorker?.terminate();
_dedicatedWorker = null;
}
return _ProbeResult(availableImplementations, existingDatabases.toList(),
missingFeatures, this);
}
Future<void> _probeDedicated() async {
@ -119,9 +131,40 @@ class WasmDatabaseOpener2 {
missingFeatures.add(MissingBrowserFeature.dedicatedWorkers);
}
}
Future<void> _probeShared() async {
if (supportsSharedWorkers) {
final sharedWorker =
SharedWorker(driftWorkerUri.toString(), 'drift worker');
final port = _sharedWorker = sharedWorker.port!;
final sharedMessages =
StreamQueue(_readMessages(port.onMessage, sharedWorker.onError));
// First, the shared worker will tell us which features it supports.
_createCompatibilityCheck().sendToPort(port);
final sharedFeatures =
await sharedMessages.nextNoError as SharedWorkerCompatibilityResult;
await sharedMessages.cancel();
_handleCompatibilityResult(sharedFeatures);
// Prefer to use the shared worker to host the database if it supports the
// necessary APIs.
if (sharedFeatures.canSpawnDedicatedWorkers &&
sharedFeatures.dedicatedWorkersCanUseOpfs) {
availableImplementations.add(WasmStorageImplementation.opfsShared);
}
if (sharedFeatures.canUseIndexedDb) {
availableImplementations.add(WasmStorageImplementation.sharedIndexedDb);
}
} else {
missingFeatures.add(MissingBrowserFeature.sharedWorkers);
}
}
}
final class _ProbeResult extends WasmProbeResult {
final class _ProbeResult implements WasmProbeResult {
@override
final List<WasmStorageImplementation> availableStorages;
@ -131,7 +174,7 @@ final class _ProbeResult extends WasmProbeResult {
@override
final Set<MissingBrowserFeature> missingFeatures;
final WasmDatabaseOpener2 opener;
final WasmDatabaseOpener opener;
_ProbeResult(
this.availableStorages,
@ -142,124 +185,27 @@ final class _ProbeResult extends WasmProbeResult {
@override
Future<DatabaseConnection> open(
WasmStorageImplementation implementation, String name,
{FutureOr<Uint8List?> Function()? initializeDatabase}) async {
// TODO: implement open
throw UnimplementedError();
}
@override
Future<void> deleteDatabase(
DatabaseLocation implementation, String name) async {
// TODO: implement deleteDatabase
throw UnimplementedError();
}
}
class WasmDatabaseOpener {
final Uri sqlite3WasmUri;
final Uri driftWorkerUri;
final String databaseName;
FutureOr<Uint8List?> Function()? initializeDatabase;
final Set<MissingBrowserFeature> missingFeatures = {};
final List<WasmStorageImplementation> availableImplementations = [
WasmStorageImplementation.inMemory,
];
bool _existsInIndexedDb = false;
bool _existsInOpfs = false;
MessagePort? _sharedWorker;
Worker? _dedicatedWorker;
WasmDatabaseOpener({
required this.sqlite3WasmUri,
required this.driftWorkerUri,
required this.databaseName,
this.initializeDatabase,
});
Future<void> probe() async {
try {
await _probeShared();
} on Object {
_sharedWorker?.close();
_sharedWorker = null;
}
try {
await _probeDedicated();
} on Object {
_dedicatedWorker?.terminate();
_dedicatedWorker = null;
}
if (_dedicatedWorker == null) {
// Something is wrong with web workers, let's see if we can get things
// running without a worker.
if (await checkIndexedDbSupport()) {
availableImplementations.add(WasmStorageImplementation.unsafeIndexedDb);
_existsInIndexedDb = await checkIndexedDbExists(databaseName);
}
}
}
Future<WasmDatabaseResult> open() async {
await probe();
// If we have an existing database in storage, we want to keep using that
// format to avoid data loss (e.g. after a browser update that enables a
// otherwise preferred storage implementation). In the future, we might want
// to consider migrating between storage implementations as well.
if (_existsInIndexedDb &&
(availableImplementations
.contains(WasmStorageImplementation.sharedIndexedDb) ||
availableImplementations
.contains(WasmStorageImplementation.unsafeIndexedDb))) {
availableImplementations.removeWhere((element) =>
element != WasmStorageImplementation.sharedIndexedDb &&
element != WasmStorageImplementation.unsafeIndexedDb);
} else if (_existsInOpfs &&
(availableImplementations
.contains(WasmStorageImplementation.opfsShared) ||
availableImplementatioobjectns
.contains(WasmStorageImplementation.opfsLocks))) {
availableImplementations.removeWhere((element) =>
element != WasmStorageImplementation.opfsShared &&
element != WasmStorageImplementation.opfsLocks);
}
// Enum values are ordered by preferrability, so just pick the best option
// left.
availableImplementations.sortBy<num>((element) => element.index);
return await _connect(availableImplementations.firstOrNull ??
WasmStorageImplementation.inMemory);
}
/// Opens a database with the given [storage] implementation, bypassing the
/// feature detection. Must be called after [probe].
Future<WasmDatabaseResult> openWith(WasmStorageImplementation storage) async {
return await _connect(storage);
}
Future<WasmDatabaseResult> _connect(WasmStorageImplementation storage) async {
WasmStorageImplementation implementation,
String name, {
FutureOr<Uint8List?> Function()? initializeDatabase,
}) async {
final channel = MessageChannel();
final initializer = initializeDatabase;
final initChannel = initializer != null ? MessageChannel() : null;
final local = channel.port1.channel();
final message = ServeDriftDatabase(
sqlite3WasmUri: sqlite3WasmUri,
sqlite3WasmUri: opener.sqlite3WasmUri,
port: channel.port2,
storage: storage,
databaseName: databaseName,
storage: implementation,
databaseName: name,
initializationPort: initChannel?.port2,
);
final sharedWorker = _sharedWorker;
final dedicatedWorker = _dedicatedWorker;
final sharedWorker = opener._sharedWorker;
final dedicatedWorker = opener._dedicatedWorker;
switch (storage) {
switch (implementation) {
case WasmStorageImplementation.opfsShared:
case WasmStorageImplementation.sharedIndexedDb:
// These are handled by the shared worker, so we can close the dedicated
@ -275,14 +221,14 @@ class WasmDatabaseOpener {
} else {
// Workers seem to be broken, but we don't need them with this storage
// mode.
return _hostDatabaseLocally(
storage, await IndexedDbFileSystem.open(dbName: databaseName));
return _hostDatabaseLocally(implementation,
await IndexedDbFileSystem.open(dbName: name), initializeDatabase);
}
case WasmStorageImplementation.inMemory:
// Nothing works on this browser, so we'll fall back to an in-memory
// database.
return _hostDatabaseLocally(storage, InMemoryFileSystem());
return _hostDatabaseLocally(
implementation, InMemoryFileSystem(), initializeDatabase);
}
initChannel?.port1.onMessage.listen((event) async {
@ -299,7 +245,7 @@ class WasmDatabaseOpener {
});
var connection = await connectToRemoteAndInitialize(local);
if (storage == WasmStorageImplementation.opfsLocks) {
if (implementation == WasmStorageImplementation.opfsLocks) {
// We want stream queries to update for writes in other tabs. For the
// implementations backed by a shared worker, the worker takes care of
// that.
@ -311,20 +257,21 @@ class WasmDatabaseOpener {
connection = DatabaseConnection(
connection.executor,
connectionData: connection.connectionData,
streamQueries: BroadcastStreamQueryStore(databaseName),
streamQueries: BroadcastStreamQueryStore(name),
);
}
}
return WasmDatabaseResult(connection, storage, missingFeatures);
return connection;
}
/// Returns a database connection that doesn't use web workers.
Future<WasmDatabaseResult> _hostDatabaseLocally(
WasmStorageImplementation storage, VirtualFileSystem vfs) async {
final initializer = initializeDatabase;
final sqlite3 = await WasmSqlite3.loadFromUrl(sqlite3WasmUri);
Future<DatabaseConnection> _hostDatabaseLocally(
WasmStorageImplementation storage,
VirtualFileSystem vfs,
FutureOr<Uint8List?> Function()? initializer,
) async {
final sqlite3 = await WasmSqlite3.loadFromUrl(opener.sqlite3WasmUri);
sqlite3.registerVirtualFileSystem(vfs);
if (initializer != null) {
@ -338,76 +285,16 @@ class WasmDatabaseOpener {
}
}
return WasmDatabaseResult(
DatabaseConnection(
WasmDatabase(sqlite3: sqlite3, path: '/database'),
),
storage,
missingFeatures,
return DatabaseConnection(
WasmDatabase(sqlite3: sqlite3, path: '/database'),
);
}
Future<void> _probeShared() async {
if (supportsSharedWorkers) {
final sharedWorker =
SharedWorker(driftWorkerUri.toString(), 'drift worker');
final port = _sharedWorker = sharedWorker.port!;
final sharedMessages =
StreamQueue(_readMessages(port.onMessage, sharedWorker.onError));
// First, the shared worker will tell us which features it supports.
RequestCompatibilityCheck(databaseName).sendToPort(port);
final sharedFeatures =
await sharedMessages.nextNoError as SharedWorkerCompatibilityResult;
await sharedMessages.cancel();
missingFeatures.addAll(sharedFeatures.missingFeatures);
_existsInOpfs |= sharedFeatures.opfsExists;
_existsInIndexedDb |= sharedFeatures.indexedDbExists;
// Prefer to use the shared worker to host the database if it supports the
// necessary APIs.
if (sharedFeatures.canSpawnDedicatedWorkers &&
sharedFeatures.dedicatedWorkersCanUseOpfs) {
availableImplementations.add(WasmStorageImplementation.opfsShared);
}
if (sharedFeatures.canUseIndexedDb) {
availableImplementations.add(WasmStorageImplementation.sharedIndexedDb);
}
} else {
missingFeatures.add(MissingBrowserFeature.sharedWorkers);
}
}
Future<void> _probeDedicated() async {
if (supportsWorkers) {
final dedicatedWorker =
_dedicatedWorker = Worker(driftWorkerUri.toString());
RequestCompatibilityCheck(databaseName).sendToWorker(dedicatedWorker);
final workerMessages = StreamQueue(
_readMessages(dedicatedWorker.onMessage, dedicatedWorker.onError));
final status = await workerMessages.nextNoError
as DedicatedWorkerCompatibilityResult;
missingFeatures.addAll(status.missingFeatures);
_existsInOpfs |= status.opfsExists;
_existsInIndexedDb |= status.indexedDbExists;
if (status.supportsNestedWorkers &&
status.canAccessOpfs &&
status.supportsSharedArrayBuffers) {
availableImplementations.add(WasmStorageImplementation.opfsLocks);
}
if (status.supportsIndexedDb) {
availableImplementations.add(WasmStorageImplementation.unsafeIndexedDb);
}
} else {
missingFeatures.add(MissingBrowserFeature.dedicatedWorkers);
}
@override
Future<void> deleteDatabase(
DatabaseLocation implementation, String name) async {
// TODO: implement deleteDatabase
throw UnimplementedError();
}
}

View File

@ -103,7 +103,8 @@ final class SharedWorkerCompatibilityResult extends CompatibilityResult {
final List<(DatabaseLocation, String)> existingDatabases;
if (asList.length > 5) {
existingDatabases = EncodeLocations.readFromJs(asList[5] as JsArray);
existingDatabases =
EncodeLocations.readFromJs(asList[5] as List<dynamic>);
} else {
existingDatabases = const [];
}
@ -300,7 +301,7 @@ final class DedicatedWorkerCompatibilityResult extends CompatibilityResult {
}
extension EncodeLocations on List<(DatabaseLocation, String)> {
static List<(DatabaseLocation, String)> readFromJs(JsArray object) {
static List<(DatabaseLocation, String)> readFromJs(List<Object?> object) {
final existing = <(DatabaseLocation, String)>[];
for (final entry in object) {

View File

@ -11,11 +11,11 @@ import 'dart:html';
import 'dart:typed_data';
import 'package:collection/collection.dart';
import 'package:drift/src/web/wasm_setup.dart';
import 'package:sqlite3/wasm.dart';
import 'backends.dart';
import 'src/sqlite3/database.dart';
import 'src/web/wasm_setup.dart';
import 'src/web/wasm_setup/dedicated_worker.dart';
import 'src/web/wasm_setup/shared_worker.dart';
import 'src/web/wasm_setup/types.dart';
@ -92,8 +92,11 @@ class WasmDatabase extends DelegatedDatabase {
required Uri driftWorkerUri,
FutureOr<Uint8List?> Function()? initializeDatabase,
}) async {
final probed =
await probe(sqlite3Uri: sqlite3Uri, driftWorkerUri: driftWorkerUri);
final probed = await probe(
sqlite3Uri: sqlite3Uri,
driftWorkerUri: driftWorkerUri,
databaseName: databaseName,
);
// If we have an existing database in storage, we want to keep using that
// format to avoid data loss (e.g. after a browser update that enables a
@ -137,10 +140,35 @@ class WasmDatabase extends DelegatedDatabase {
connection, bestImplementation, probed.missingFeatures);
}
/// Probes for:
///
/// - available storage implementations based on supported web APIs.
/// - APIs not currently supported by the browser.
/// - existing drift databases in the current browsing context.
///
/// This information can be used to control whether to open a drift database,
/// or whether the current browser is unsuitable for the persistence
/// requirements of your app.
/// For most apps, using [open] directly is easier. It calls [probe]
/// internally and uses the best storage implementation available.
///
/// The [databaseName] option is not strictly required. But drift can't list
/// databases stored in IndexedDb, they are not part of
/// [WasmProbeResult.existingDatabases] by default. This is because drift
/// databases can't be distinguished from other IndexedDb databases without
/// opening them, which might disturb the running operation of them. When a
/// [databaseName] is passed, drift will explicitly probe whether a database
/// with that name exists in IndexedDb and whether it is a drift database.
/// Drift is always able to list databases stored in OPFS, regardless of
/// whether [databaseName] is passed or not.
static Future<WasmProbeResult> probe({
required Uri sqlite3Uri,
required Uri driftWorkerUri,
}) {}
String? databaseName,
}) async {
return await WasmDatabaseOpener(sqlite3Uri, driftWorkerUri, databaseName)
.probe();
}
/// The entrypoint for a web worker suitable for use with [open].
///

View File

@ -1,18 +1,7 @@
Integration tests for `package:drift/native.dart`.
To test persistence, we need to instrument a browsers in a way not covered by the normal
`test` package. For instance, we need to reload pages to ensure data is still there.
To run the tests automatically (with us managing a browser driver), just run `dart test`.
## Running tests with Firefox
```
geckodriver &
dart run tool/drift_wasm_test.dart firefox http://localhost:4444
```
## Running tests with Chrome
```
chromedriver --port=4444 --url-base=wd/hub &
dart run tool/drift_wasm_test.dart chrome http://localhost:4444/wd/hub/
```
To manually debug issues, it might make sense to trigger some functionality manually.
You can run `dart run tool/server_manually.dart` to start a web server hosting the test
content on http://localhost:8080.

View File

@ -5,14 +5,15 @@ import 'dart:js_util';
import 'package:async/async.dart';
import 'package:drift/drift.dart';
import 'package:drift/wasm.dart';
// ignore: invalid_use_of_internal_member
import 'package:drift/src/web/wasm_setup.dart';
import 'package:http/http.dart' as http;
import 'package:web_wasm/initialization_mode.dart';
import 'package:web_wasm/src/database.dart';
import 'package:sqlite3/wasm.dart';
const dbName = 'drift_test';
final sqlite3WasmUri = Uri.parse('/sqlite3.wasm');
final driftWorkerUri = Uri.parse('/worker.dart.js');
TestDatabase? openedDatabase;
StreamQueue<void>? tableUpdates;
@ -31,7 +32,12 @@ void main() {
document.getElementById('selfcheck')?.onClick.listen((event) async {
print('starting');
final database = await _opener.open();
final database = await WasmDatabase.open(
databaseName: dbName,
sqlite3Uri: sqlite3WasmUri,
driftWorkerUri: driftWorkerUri,
initializeDatabase: _initializeDatabase,
);
print('selected storage: ${database.chosenImplementation}');
print('missing features: ${database.missingFeatures}');
@ -54,77 +60,80 @@ void _addCallbackForWebDriver(String name, Future Function(String?) impl) {
}));
}
WasmDatabaseOpener get _opener {
Future<Uint8List> Function()? initializeDatabase;
Future<Uint8List?> _initializeDatabase() async {
switch (initializationMode) {
case InitializationMode.loadAsset:
initializeDatabase = () async {
final response = await http.get(Uri.parse('/initial.db'));
return response.bodyBytes;
};
final response = await http.get(Uri.parse('/initial.db'));
return response.bodyBytes;
case InitializationMode.migrateCustomWasmDatabase:
initializeDatabase = () async {
// Let's first open a custom WasmDatabase, the way it would have been
// done before WasmDatabase.open.
final sqlite3 =
await WasmSqlite3.loadFromUrl(Uri.parse('/sqlite3.wasm'));
final fs = await IndexedDbFileSystem.open(dbName: dbName);
sqlite3.registerVirtualFileSystem(fs, makeDefault: true);
final wasmDb = WasmDatabase(sqlite3: sqlite3, path: '/app.db');
final db = TestDatabase(wasmDb);
await db
.into(db.testTable)
.insert(TestTableCompanion.insert(content: 'from old database'));
await db.close();
// Let's first open a custom WasmDatabase, the way it would have been
// done before WasmDatabase.open.
final sqlite3 = await WasmSqlite3.loadFromUrl(Uri.parse('/sqlite3.wasm'));
final fs = await IndexedDbFileSystem.open(dbName: dbName);
sqlite3.registerVirtualFileSystem(fs, makeDefault: true);
final (file: file, outFlags: _) =
fs.xOpen(Sqlite3Filename('/app.db'), 0);
final blob = Uint8List(file.xFileSize());
file.xRead(blob, 0);
file.xClose();
fs.xDelete('/app.db', 0);
await fs.close();
final wasmDb = WasmDatabase(sqlite3: sqlite3, path: '/app.db');
final db = TestDatabase(wasmDb);
await db
.into(db.testTable)
.insert(TestTableCompanion.insert(content: 'from old database'));
await db.close();
return blob;
};
break;
final (file: file, outFlags: _) = fs.xOpen(Sqlite3Filename('/app.db'), 0);
final blob = Uint8List(file.xFileSize());
file.xRead(blob, 0);
file.xClose();
fs.xDelete('/app.db', 0);
await fs.close();
return blob;
case InitializationMode.none:
break;
return null;
}
return WasmDatabaseOpener(
databaseName: dbName,
sqlite3WasmUri: Uri.parse('/sqlite3.wasm'),
driftWorkerUri: Uri.parse('/worker.dart.js'),
initializeDatabase: initializeDatabase,
);
}
Future<String> _detectImplementations(String? _) async {
final opener = _opener;
await opener.probe();
final result = await WasmDatabase.probe(
sqlite3Uri: sqlite3WasmUri,
driftWorkerUri: driftWorkerUri,
databaseName: dbName,
);
return json.encode({
'impls': opener.availableImplementations.map((r) => r.name).toList(),
'missing': opener.missingFeatures.map((r) => r.name).toList(),
'impls': result.availableStorages.map((r) => r.name).toList(),
'missing': result.missingFeatures.map((r) => r.name).toList(),
});
}
Future<void> _open(String? implementationName) async {
final opener = _opener;
WasmDatabaseResult result;
DatabaseConnection connection;
if (implementationName != null) {
await opener.probe();
result = await opener
.openWith(WasmStorageImplementation.values.byName(implementationName));
final probeResult = await WasmDatabase.probe(
sqlite3Uri: sqlite3WasmUri,
driftWorkerUri: driftWorkerUri,
databaseName: dbName,
);
connection = await probeResult.open(
WasmStorageImplementation.values.byName(implementationName),
dbName,
initializeDatabase: _initializeDatabase,
);
} else {
result = await opener.open();
final result = await WasmDatabase.open(
databaseName: dbName,
sqlite3Uri: sqlite3WasmUri,
driftWorkerUri: driftWorkerUri,
initializeDatabase: _initializeDatabase,
);
connection = result.resolvedExecutor;
}
final db = openedDatabase = TestDatabase(result.resolvedExecutor);
final db = openedDatabase = TestDatabase(connection);
// Make sure it works!
await db.customSelect('SELECT 1').get();