Add script to build a local sqlite3 library

This local library is then used in drift tests by default, making it
easier to run drift unit tests against the expected sqlite3 version.

.github/workflows/main.yml

@@ -9,38 +9,36 @@ jobs:
   # Compile the latest sqlite3 library, which will be used to run tests in drift
   # and sqlparser
   compile_sqlite3:
+    strategy:
+      matrix:
+        # We only really need this for Ubuntu, but we recommend users run the same
+        # steps so we better make sure they work on all platforms.
+        os: [ubuntu-latest, macos-latest, windows-latest]
     name: "Compile sqlite3 for tests"
-    runs-on: ubuntu-latest
-    env:
-      SQLITE_YEAR: "2022"
-      SQLITE_VERSION: "3390300"
-
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
       - uses: actions/cache@v3
         id: cache_build
         with:
-          path: /tmp/sqlite/out/
-          key: ${{ runner.os }}-${{ env.SQLITE_VERSION }}
+          path: drift/.dart_tool/sqlite3/
+          key: ${{ runner.os }}-${{ hashFiles('drift/tool/') }}
+      - name: Download Dart
+        if: steps.cache_build.outputs.cache-hit != 'true'
+        uses: dart-lang/setup-dart@v1
       - name: Compile sqlite3
         if: steps.cache_build.outputs.cache-hit != 'true'
         run: |
-          cd /tmp/
-          mkdir sqlite
-          cd sqlite
-          curl https://sqlite.org/$SQLITE_YEAR/sqlite-autoconf-$SQLITE_VERSION.tar.gz --output sqlite.tar.gz
-          tar zxvf sqlite.tar.gz
-          cd sqlite-autoconf-$SQLITE_VERSION
-          ./configure
-          make
-          mkdir ../out
-          cp sqlite3 ../out
-          cp .libs/libsqlite3.so ../out
+          dart pub global activate melos
+          melos bootstrap --scope drift
+          dart run drift/tool/download_sqlite3.dart
       - name: Upload built sqlite3 binaries
         uses: actions/upload-artifact@v2
+        # we only need these artifacts on Linux since we run unit tests on Linux only
+        if: runner.os == 'Linux'
         with:
           name: sqlite3
-          path: /tmp/sqlite/out/
+          path: drift/.dart_tool/sqlite3/
           if-no-files-found: error
           retention-days: 1
 
@@ -156,10 +154,10 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           name: sqlite3
-          path: /tmp/sqlite/out/
+          path: drift/.dart_tool/sqlite3/
       - name: Use downloaded sqlite3
         run: |
-          chmod a+x /tmp/sqlite/out/sqlite3
+          chmod a+x .dart_tool/sqlite3//sqlite3
           echo "/tmp/sqlite/out" >> $GITHUB_PATH
           echo "LD_LIBRARY_PATH=/tmp/sqlite/out" >> $GITHUB_ENV
       - name: Check sqlite3 version

drift/pubspec.yaml

@@ -18,6 +18,7 @@ dependencies:
   sqlite3: ^1.7.1
 
 dev_dependencies:
+  archive: ^3.3.1
   build_test: ^2.0.0
   build_runner_core: ^7.0.0
   build_verify: ^3.0.0

drift/test/engines/connection_pool_integration_test.dart

@@ -8,6 +8,7 @@ import 'package:path/path.dart' show join;
 import 'package:test/test.dart';
 
 import '../generated/todos.dart';
+import '../test_utils/database_vm.dart';
 
 String fileName = 'drift-wal-integration-test.db';
 final _file = File(join(Directory.systemTemp.path, fileName));
@@ -19,6 +20,8 @@ DatabaseConnection _forBackgroundIsolate() {
 }
 
 void main() {
+  preferLocalSqlite3();
+
   setUp(() async {
     if (await _file.exists()) {
       await _file.delete();

drift/test/integration_tests/cancellation_test.dart

@@ -4,9 +4,11 @@ import 'package:drift/drift.dart';
 import 'package:drift/isolate.dart';
 import 'package:test/test.dart';
 
+import '../test_utils/database_vm.dart';
 import 'cancellation_test_support.dart';
 
 void main() {
+  preferLocalSqlite3();
   driftRuntimeOptions.dontWarnAboutMultipleDatabases = true;
 
   Future<void> runTest(EmptyDb db) async {

drift/test/integration_tests/common_ffi_test.dart

@@ -7,6 +7,8 @@ import 'package:path/path.dart' show join;
 import 'package:sqlite3/sqlite3.dart' hide Database;
 import 'package:test/test.dart';
 
+import '../test_utils/database_vm.dart';
+
 class DriftNativeExcecutor extends TestExecutor {
   static String fileName =
       'drift-native-tests-${DateTime.now().toIso8601String()}';
@@ -35,6 +37,8 @@ class DriftNativeExcecutor extends TestExecutor {
 }
 
 void main() {
+  preferLocalSqlite3();
+
   runAllTests(DriftNativeExcecutor());
 
   test('can save and restore a database', () async {

drift/test/integration_tests/database_test.dart

@@ -6,8 +6,11 @@ import 'package:sqlite3/sqlite3.dart';
 import 'package:test/test.dart';
 
 import '../generated/todos.dart';
+import '../test_utils/database_vm.dart';
 
 void main() {
+  preferLocalSqlite3();
+
   test('transaction handles BEGIN throwing', () async {
     final rawDb = sqlite3.open('file:transaction_test?mode=memory&cache=shared',
         uri: true);

drift/test/integration_tests/migrations_integration_test.dart

@@ -7,8 +7,11 @@ import 'package:test/test.dart';
 
 import '../generated/custom_tables.dart';
 import '../generated/todos.dart';
+import '../test_utils/database_vm.dart';
 
 void main() {
+  preferLocalSqlite3();
+
   test('change column types', () async {
     // Create todos table with category as text (it's an int? in Dart).
     final executor = NativeDatabase.memory(setup: (db) {

drift/test/integration_tests/regress_1589_test.dart

@@ -3,7 +3,11 @@ import 'package:drift/drift.dart';
 import 'package:drift/native.dart';
 import 'package:test/test.dart';
 
+import '../test_utils/database_vm.dart';
+
 void main() {
+  preferLocalSqlite3();
+
   test('a failing commit does not block the whole database', () async {
     final db = _Database(NativeDatabase.memory());
     addTearDown(db.close);

drift/test/integration_tests/rx_cancellation_test.dart

@@ -4,9 +4,12 @@ import 'package:drift/isolate.dart';
 import 'package:rxdart/rxdart.dart';
 import 'package:test/test.dart';
 
+import '../test_utils/database_vm.dart';
 import 'cancellation_test_support.dart';
 
 void main() {
+  preferLocalSqlite3();
+
   test('together with switchMap', () async {
     String slowQuery(int i) => '''
       with recursive slow(x) as (values(log_value($i)) union all select x+1 from slow where x < 1000000)

drift/test/isolate_test.dart

@@ -9,9 +9,11 @@ import 'package:mockito/mockito.dart';
 import 'package:test/test.dart';
 
 import 'generated/todos.dart';
+import 'test_utils/database_vm.dart';
 import 'test_utils/test_utils.dart';
 
 void main() {
+  preferLocalSqlite3();
   // Using the DriftIsolate apis without actually running on a background
   // isolate is pointless, but we can't collect coverage for background
   // isolates: https://github.com/dart-lang/test/issues/1108

drift/test/platforms/ffi/moor_functions_test.dart

@@ -5,7 +5,11 @@ import 'package:drift/src/sqlite3/native_functions.dart';
 import 'package:sqlite3/sqlite3.dart';
 import 'package:test/test.dart';
 
+import '../../test_utils/database_vm.dart';
+
 void main() {
+  preferLocalSqlite3();
+
   late Database db;
 
   setUp(() => db = sqlite3.openInMemory()..useNativeFunctions());

drift/test/platforms/ffi/native_database_test.dart

@@ -4,7 +4,11 @@ import 'package:drift/native.dart';
 import 'package:sqlite3/sqlite3.dart';
 import 'package:test/test.dart';
 
+import '../../test_utils/database_vm.dart';
+
 void main() {
+  preferLocalSqlite3();
+
   group('NativeDatabase.opened', () {
     test('disposes the underlying database by default', () async {
       final underlying = sqlite3.openInMemory();

drift/test/platforms/ffi/setup_test.dart

@@ -4,6 +4,7 @@ import 'package:sqlite3/sqlite3.dart';
 import 'package:test/test.dart';
 
 import '../../generated/todos.dart';
+import '../../test_utils/database_vm.dart';
 
 void _setup(Database db) {
   db.createFunction(
@@ -13,6 +14,8 @@ void _setup(Database db) {
 }
 
 void main() {
+  preferLocalSqlite3();
+
   test('can use a custom setup function', () async {
     final executor = NativeDatabase.memory(setup: _setup);
 

drift/test/remote_test.dart

@@ -12,6 +12,8 @@ import 'test_utils/database_vm.dart';
 import 'test_utils/mocks.dart';
 
 void main() {
+  preferLocalSqlite3();
+
   test('closes channel in shutdown', () async {
     final controller = StreamChannelController();
     final server =

drift/test/test_utils/database_vm.dart

@@ -1,11 +1,53 @@
+import 'dart:ffi';
+import 'dart:io';
+
 import 'package:drift/drift.dart';
 import 'package:drift/native.dart';
 import 'package:sqlite3/sqlite3.dart';
+import 'package:sqlite3/open.dart';
+import 'package:path/path.dart' as p;
+
+bool _checkedForLocalSqlite3 = false;
+
+String? get expectedLocalSqlite3Path {
+  if (Platform.isWindows) {
+    return p.join('.dart_tool', 'sqlite3', 'sqlite3.dll');
+  } else if (Platform.isMacOS) {
+    return p.join('.dart_tool', 'sqlite3', 'libsqlite3.dylib');
+  } else if (Platform.isLinux) {
+    return p.join('.dart_tool', 'sqlite3', 'libsqlite3.so');
+  } else {
+    return null;
+  }
+}
+
+/// Checks if a sqlite3 build has been downloaded into [expectedLocalSqlite3Path],
+/// usually by the user running `dart run tool/download_sqlite3.dart` before
+/// running tests.
+///
+/// If such file exists, we prefer it over the (potentially outdated) system's
+/// sqlite3.
+///
+/// This needs to be called before using sqlite3 in a test.
+void preferLocalSqlite3() {
+  if (!_checkedForLocalSqlite3) {
+    _checkedForLocalSqlite3 = true;
+
+    final path = expectedLocalSqlite3Path;
+    if (path == null) return;
+
+    if (File(path).existsSync()) {
+      open.overrideForAll(() => DynamicLibrary.open(p.absolute(path)));
+    }
+  }
+}
 
 Version get sqlite3Version {
+  preferLocalSqlite3();
   return sqlite3.version;
 }
 
 DatabaseConnection testInMemoryDatabase() {
+  preferLocalSqlite3();
   return DatabaseConnection(NativeDatabase.memory());
 }

drift/test/test_utils/has_local_sqlite3_test.dart

@@ -0,0 +1,19 @@
+import 'dart:io';
+
+import 'package:test/test.dart';
+
+import 'database_vm.dart';
+
+void main() {
+  final path = expectedLocalSqlite3Path;
+  final printWarning = path != null && !File(path).existsSync();
+
+  test(
+    'check for local sqlite3 library',
+    () {},
+    skip: printWarning
+        ? 'Local sqlite3 library for drift tests does not exist, falling back '
+            'to the one from the OS. Please run `dart run tool/download_sqlite3.dart`'
+        : null,
+  );
+}

drift/tool/download_sqlite3.dart

@@ -0,0 +1,111 @@
+import 'dart:io';
+
+import 'package:archive/archive_io.dart';
+import 'package:http/http.dart';
+import 'package:path/path.dart' as p;
+
+const _version = '3390300';
+const _year = '2022';
+const _url = 'https://www.sqlite.org/$_year/sqlite-autoconf-$_version.tar.gz';
+
+Future<void> main(List<String> args) async {
+  if (args.contains('version')) {
+    print(_version);
+    exit(0);
+  }
+
+  final driftDirectory = p.dirname(p.dirname(Platform.script.toFilePath()));
+  final target = p.join(driftDirectory, '.dart_tool', 'sqlite3');
+  final versionFile = File(p.join(target, 'version'));
+
+  final needsDownload = args.contains('--force') ||
+      !versionFile.existsSync() ||
+      versionFile.readAsStringSync() != _version;
+
+  if (!needsDownload) {
+    print('Not doing anything as sqlite3 has already been downloaded. Use '
+        '--force to re-compile it.');
+    exit(0);
+  }
+
+  print('Downloading and compiling sqlite3 for drift test');
+
+  final temporaryDir =
+      await Directory.systemTemp.createTemp('drift-compile-sqlite3');
+  final temporaryDirPath = temporaryDir.path;
+
+  // Compiling on Windows is ugly because we need users to have Visual Studio
+  // installed and all those tools activated in the current shell.
+  // Much easier to just download precompiled builds.
+  if (Platform.isWindows) {
+    const windowsUri =
+        'https://www.sqlite.org/$_year/sqlite-dll-win64-x64-$_version.zip';
+    final sqlite3Zip = p.join(temporaryDirPath, 'sqlite3.zip');
+    final client = Client();
+    final response = await client.send(Request('GET', Uri.parse(windowsUri)));
+    if (response.statusCode != 200) {
+      print(
+          'Could not download $windowsUri, status code ${response.statusCode}');
+      exit(1);
+    }
+    await response.stream.pipe(File(sqlite3Zip).openWrite());
+
+    final inputStream = InputFileStream(sqlite3Zip);
+    final archive = ZipDecoder().decodeBuffer(inputStream);
+
+    for (final file in archive.files) {
+      if (file.isFile && file.name == 'sqlite3.dll') {
+        final outputStream = OutputFileStream(p.join(target, 'sqlite3.dll'));
+
+        file.writeContent(outputStream);
+        outputStream.close();
+      }
+    }
+
+    await File(p.join(target, 'version')).writeAsString(_version);
+    exit(0);
+  }
+
+  await _run('curl $_url --output sqlite.tar.gz',
+      workingDirectory: temporaryDirPath);
+  await _run('tar zxvf sqlite.tar.gz', workingDirectory: temporaryDirPath);
+
+  final sqlitePath = p.join(temporaryDirPath, 'sqlite-autoconf-$_version');
+  await _run('./configure', workingDirectory: sqlitePath);
+  await _run('make -j', workingDirectory: sqlitePath);
+
+  final targetDirectory = Directory(target);
+
+  if (!targetDirectory.existsSync()) {
+    // Not using recursive since .dart_tool should really exist already.
+    targetDirectory.createSync();
+  }
+
+  await File(p.join(sqlitePath, 'sqlite3')).copy(p.join(target, 'sqlite3'));
+
+  if (Platform.isLinux) {
+    await File(p.join(sqlitePath, '.libs', 'libsqlite3.so'))
+        .copy(p.join(target, 'libsqlite3.so'));
+  } else if (Platform.isMacOS) {
+    await File(p.join(sqlitePath, '.libs', 'libsqlite3.dylib'))
+        .copy(p.join(target, 'libsqlite3.dylib'));
+  }
+
+  await File(p.join(target, 'version')).writeAsString(_version);
+}
+
+Future<void> _run(String command, {String? workingDirectory}) async {
+  print('Running $command');
+
+  final proc = await Process.start(
+    'sh',
+    ['-c', command],
+    mode: ProcessStartMode.inheritStdio,
+    workingDirectory: workingDirectory,
+  );
+  final exitCode = await proc.exitCode;
+
+  if (exitCode != 0) {
+    exit(exitCode);
+  }
+}