From 7c62d6cdd17c5485a472549c50ed50bb5b0bdd3b Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 16 Feb 2020 13:26:25 +0100 Subject: [PATCH] Provide IndexedDB backend on the web (#390) --- .../tests/lib/suite/crud_tests.dart | 19 ++- .../tests/lib/suite/transactions.dart | 1 + .../web/test/initializer_test.dart | 40 +++-- .../web/test/integration_test.dart | 22 ++- moor/CHANGELOG.md | 4 + moor/lib/moor_web.dart | 2 + .../src/runtime/executor/helpers/engines.dart | 5 + moor/lib/src/web/storage.dart | 140 ++++++++++++++++++ moor/lib/src/web/web_db.dart | 76 +++++----- 9 files changed, 245 insertions(+), 64 deletions(-) create mode 100644 moor/lib/src/web/storage.dart diff --git a/extras/integration_tests/tests/lib/suite/crud_tests.dart b/extras/integration_tests/tests/lib/suite/crud_tests.dart index 8e2c94f4..534b28b4 100644 --- a/extras/integration_tests/tests/lib/suite/crud_tests.dart +++ b/extras/integration_tests/tests/lib/suite/crud_tests.dart @@ -5,23 +5,20 @@ import 'package:tests/suite/suite.dart'; void crudTests(TestExecutor executor) { test('inserting updates a select stream', () async { final db = Database(executor.createExecutor()); - final friends = db.watchFriendsOf(1); + final friends = db.watchFriendsOf(1).asBroadcastStream(); final a = await db.getUserById(1); final b = await db.getUserById(2); - final expectation = expectLater( - friends, - emitsInOrder( - [ - isEmpty, // initial state without friendships - equals([b]), // after we called makeFriends(a,b) - ], - ), - ); + expect(await friends.first, isEmpty); + + // after we called makeFriends(a,b) + final expectation = expectLater(friends, emits(equals([b]))); await db.makeFriends(a, b); await expectation; + + await db.close(); }); test('IN ? expressions can be expanded', () async { @@ -31,5 +28,7 @@ void crudTests(TestExecutor executor) { final result = await db.usersById([1, 2, 3]); expect(result.map((u) => u.name), ['Dash', 'Duke', 'Go Gopher']); + + await db.close(); }); } diff --git a/extras/integration_tests/tests/lib/suite/transactions.dart b/extras/integration_tests/tests/lib/suite/transactions.dart index 6d5b7c37..a94af60c 100644 --- a/extras/integration_tests/tests/lib/suite/transactions.dart +++ b/extras/integration_tests/tests/lib/suite/transactions.dart @@ -56,5 +56,6 @@ void transactionTests(TestExecutor executor) { test('can use no-op transactions', () async { final db = Database(executor.createExecutor()); await db.transaction(() => Future.value(null)); + await db.close(); }); } diff --git a/extras/integration_tests/web/test/initializer_test.dart b/extras/integration_tests/web/test/initializer_test.dart index e6740dba..20a9b1f0 100644 --- a/extras/integration_tests/web/test/initializer_test.dart +++ b/extras/integration_tests/web/test/initializer_test.dart @@ -182,21 +182,33 @@ AAAAAAAAAAAAAAAAAAAAAAANAQIjaGVsbG8gd29ybGQ= void main() { test('can initialize database when absent', () async { - var didCallInitializer = false; - final db = WebDatabase('name', initializer: () async { - didCallInitializer = true; - return base64.decode(_rawDataBase64.replaceAll('\n', '')); - }); - - db.databaseInfo = _FakeDatabase(db); - await db.ensureOpen(); - expect(didCallInitializer, isTrue); - - final result = await db.runSelect('SELECT * FROM foo', const []); - expect(result, [ - {'name': 'hello world'} - ]); + await _testWith(const MoorWebStorage('name')); }); + + test('can initialize database when absent - IndexedDB', () async { + await _testWith(MoorWebStorage.indexedDb('name')); + }); +} + +Future _testWith(MoorWebStorage storage) async { + var didCallInitializer = false; + final db = WebDatabase.withStorage(storage, initializer: () async { + didCallInitializer = true; + return base64.decode(_rawDataBase64.replaceAll('\n', '')); + }); + + moorRuntimeOptions.dontWarnAboutMultipleDatabases = true; + db.databaseInfo = _FakeDatabase(db); + + await db.ensureOpen(); + expect(didCallInitializer, isTrue); + + final result = await db.runSelect('SELECT * FROM foo', const []); + expect(result, [ + {'name': 'hello world'} + ]); + + await db.close(); } class _FakeDatabase extends GeneratedDatabase { diff --git a/extras/integration_tests/web/test/integration_test.dart b/extras/integration_tests/web/test/integration_test.dart index 547483d4..3ce6b0d3 100644 --- a/extras/integration_tests/web/test/integration_test.dart +++ b/extras/integration_tests/web/test/integration_test.dart @@ -20,6 +20,24 @@ class WebExecutor extends TestExecutor { } } -void main() { - runAllTests(WebExecutor()); +class WebExecutorIndexedDb extends TestExecutor { + @override + QueryExecutor createExecutor() { + return WebDatabase.withStorage(MoorWebStorage.indexedDb('foo')); + } + + @override + Future deleteData() async { + await window.indexedDB.deleteDatabase('moor_databases'); + } +} + +void main() { + group('using local storage', () { + runAllTests(WebExecutor()); + }); + + group('using IndexedDb', () { + runAllTests(WebExecutorIndexedDb()); + }); } diff --git a/moor/CHANGELOG.md b/moor/CHANGELOG.md index 821e5db6..55293f50 100644 --- a/moor/CHANGELOG.md +++ b/moor/CHANGELOG.md @@ -1,3 +1,7 @@ +## unreleased + +- Experimentally support IndexedDB to store sqlite data on the web + ## 2.4.0 - Support aggregate expressions and `group by` in the Dart api diff --git a/moor/lib/moor_web.dart b/moor/lib/moor_web.dart index 9b5c27d0..6c5013be 100644 --- a/moor/lib/moor_web.dart +++ b/moor/lib/moor_web.dart @@ -7,6 +7,7 @@ library moor_web; import 'dart:async'; import 'dart:html'; +import 'dart:indexed_db'; import 'package:meta/meta.dart'; @@ -17,4 +18,5 @@ import 'src/web/sql_js.dart'; export 'moor.dart'; +part 'src/web/storage.dart'; part 'src/web/web_db.dart'; diff --git a/moor/lib/src/runtime/executor/helpers/engines.dart b/moor/lib/src/runtime/executor/helpers/engines.dart index a1b36502..744b3f75 100644 --- a/moor/lib/src/runtime/executor/helpers/engines.dart +++ b/moor/lib/src/runtime/executor/helpers/engines.dart @@ -298,6 +298,11 @@ class DelegatedDatabase extends QueryExecutor with _ExecutorWithQueryDelegate { Future _runBeforeOpen(OpeningDetails d) { return databaseInfo.beforeOpenCallback(_BeforeOpeningExecutor(this), d); } + + @override + Future close() { + return delegate.close(); + } } /// Inside a `beforeOpen` callback, all moor apis must be available. At the same diff --git a/moor/lib/src/web/storage.dart b/moor/lib/src/web/storage.dart new file mode 100644 index 00000000..e354d1ad --- /dev/null +++ b/moor/lib/src/web/storage.dart @@ -0,0 +1,140 @@ +part of 'package:moor/moor_web.dart'; + +/// Interface to control how moor should store data on the web. +abstract class MoorWebStorage { + /// Opens the storage implementation. + Future open(); + + /// Closes the storage implementation. + /// + /// No further requests may be sent after [close] was called. + Future close(); + + /// Restore the last database version that was saved with [store]. + /// + /// If no saved data was found, returns null. + Future restore(); + + /// Store the entire database. + Future store(Uint8List data); + + /// Creates the default storage implementation that uses the local storage + /// apis. + /// + /// The [name] parameter can be used to store multiple databases. + const factory MoorWebStorage(String name) = _LocalStorageImpl; + + /// An experimental storage implementation that uses IndexedDB. + /// + /// This implementation is significantly faster than the default + /// implementation in local storage. Browsers also tend to allow more data + /// to be saved in IndexedDB. + /// However, older browsers might not support IndexedDB. + @experimental + factory MoorWebStorage.indexedDb(String name) = _IndexedDbStorage; +} + +abstract class _CustomSchemaVersionSave implements MoorWebStorage { + int /*?*/ get schemaVersion; + set schemaVersion(int value); +} + +class _LocalStorageImpl implements MoorWebStorage, _CustomSchemaVersionSave { + final String name; + + String get _persistenceKey => 'moor_db_str_$name'; + String get _versionKey => 'moor_db_version_$name'; + + const _LocalStorageImpl(this.name); + + @override + int get schemaVersion { + final versionStr = window.localStorage[_versionKey]; + // ignore: avoid_returning_null + if (versionStr == null) return null; + + return int.tryParse(versionStr); + } + + @override + set schemaVersion(int value) { + window.localStorage[_versionKey] = value.toString(); + } + + @override + Future close() => Future.value(); + + @override + Future open() => Future.value(); + + @override + Future restore() async { + final raw = window.localStorage[_persistenceKey]; + if (raw != null) { + return bin2str.decode(raw); + } + return null; + } + + @override + Future store(Uint8List data) { + final binStr = bin2str.encode(data); + window.localStorage[_persistenceKey] = binStr; + + return Future.value(); + } +} + +class _IndexedDbStorage implements MoorWebStorage { + static const _objectStoreName = 'moor_databases'; + + final String name; + + Database _database; + + _IndexedDbStorage(this.name); + + @override + Future open() async { + _database = await window.indexedDB.open( + _objectStoreName, + version: 1, + onUpgradeNeeded: (event) { + final database = event.target.result as Database; + database.createObjectStore(_objectStoreName); + }, + ); + } + + @override + Future close() async { + _database.close(); + } + + @override + Future store(Uint8List data) async { + final transaction = + _database.transactionStore(_objectStoreName, 'readwrite'); + final store = transaction.objectStore(_objectStoreName); + + await store.put(Blob([data]), name); + await transaction.completed; + } + + @override + Future restore() async { + final transaction = + _database.transactionStore(_objectStoreName, 'readonly'); + final store = transaction.objectStore(_objectStoreName); + + final result = await store.getObject(name) as Blob /*?*/; + if (result == null) return null; + + final reader = FileReader(); + reader.readAsArrayBuffer(result); + // todo: Do we need to handle errors? We're reading from memory + await reader.onLoad.first; + + return reader.result as Uint8List; + } +} diff --git a/moor/lib/src/web/web_db.dart b/moor/lib/src/web/web_db.dart index 08814f20..7a0aa6e8 100644 --- a/moor/lib/src/web/web_db.dart +++ b/moor/lib/src/web/web_db.dart @@ -14,20 +14,28 @@ class WebDatabase extends DelegatedDatabase { /// [initializer] can be used to initialize the database if it doesn't exist. WebDatabase(String name, {bool logStatements = false, CreateWebDatabase initializer}) - : super(_WebDelegate(name, initializer), + : super(_WebDelegate(MoorWebStorage(name), initializer), + logStatements: logStatements, isSequential: true); + + /// A database executor that works on the web. + /// + /// The [storage] parameter controls how the data will be stored. The default + /// constructor of [MoorWebStorage] will use local storage for that, but an + /// IndexedDB-based implementation is available via. + WebDatabase.withStorage(MoorWebStorage storage, + {bool logStatements = false, CreateWebDatabase initializer}) + : super(_WebDelegate(storage, initializer), logStatements: logStatements, isSequential: true); } class _WebDelegate extends DatabaseDelegate { - final String name; + final MoorWebStorage storage; final CreateWebDatabase initializer; SqlJsDatabase _db; - String get _persistenceKey => 'moor_db_str_$name'; - bool _inTransaction = false; - _WebDelegate(this.name, this.initializer); + _WebDelegate(this.storage, this.initializer); @override set isInTransaction(bool value) { @@ -59,11 +67,13 @@ class _WebDelegate extends DatabaseDelegate { assert(dbVersion >= 1, 'Database schema version needs to be at least 1'); final module = await initSqlJs(); - var restored = _restoreDb(); + + await storage.open(); + var restored = await storage.restore(); if (restored == null && initializer != null) { restored = await initializer(); - _storeData(restored); + await storage.store(restored); } _db = module.createDatabase(restored); @@ -123,52 +133,37 @@ class _WebDelegate extends DatabaseDelegate { } @override - Future close() { - _storeDb(); + Future close() async { + await _storeDb(); _db?.close(); - return Future.value(); + await storage.close(); } @override void notifyDatabaseOpened(OpeningDetails details) { - if (details.hadUpgrade | details.wasCreated) { + if (details.hadUpgrade || details.wasCreated) { _storeDb(); } } /// Saves the database if the last statement changed rows. As a side-effect, /// saving the database resets the `last_insert_id` counter in sqlite. - Future _handlePotentialUpdate() { + Future _handlePotentialUpdate() async { final modified = _db.lastModifiedRows(); if (modified > 0) { - _storeDb(); + await _storeDb(); } - return Future.value(modified); + return modified; } - Uint8List _restoreDb() { - final raw = window.localStorage[_persistenceKey]; - if (raw != null) { - return bin2str.decode(raw); - } - return null; - } - - void _storeDb() { + Future _storeDb() async { if (!isInTransaction) { - _storeData(_db.export()); + await storage.store(_db.export()); } } - - void _storeData(Uint8List data) { - final binStr = bin2str.encode(data); - window.localStorage[_persistenceKey] = binStr; - } } class _WebVersionDelegate extends DynamicVersionDelegate { - String get _versionKey => 'moor_db_version_${delegate.name}'; - final _WebDelegate delegate; _WebVersionDelegate(this.delegate); @@ -179,18 +174,23 @@ class _WebVersionDelegate extends DynamicVersionDelegate { @override Future get schemaVersion async { - if (!window.localStorage.containsKey(_versionKey)) { - return delegate._db?.userVersion; + final storage = delegate.storage; + int version; + if (storage is _CustomSchemaVersionSave) { + version = storage.schemaVersion; } - final versionStr = window.localStorage[_versionKey]; - return int.tryParse(versionStr); + return version ?? delegate._db.userVersion; } @override - Future setSchemaVersion(int version) { + Future setSchemaVersion(int version) async { + final storage = delegate.storage; + + if (storage is _CustomSchemaVersionSave) { + storage.schemaVersion = version; + } + delegate._db.userVersion = version; - window.localStorage[_versionKey] = version.toString(); - return Future.value(); } }