Recognize database files when dumping schema data

This commit is contained in:
Simon Binder 2022-12-15 16:47:44 +01:00
parent e5371afe92
commit 5f8b1e3358
No known key found for this signature in database
GPG Key ID: 7891917E4147B8C0
9 changed files with 469 additions and 151 deletions

View File

@ -312,6 +312,12 @@ If drift is unable to extract the version from your `schemaVersion` getter, prov
$ dart run drift_dev schema dump lib/database/database.dart drift_schemas/drift_schema_v3.json
```
{% block "blocks/alert" title='<i class="fas fa-lightbulb"></i> Dumping a database' color="success" %}
If, instead of exporting the schema of a database class, you want to export the schema of an existing sqlite3
database file, you can do that as well! `drift_dev schema dump` recognizes a sqlite3 database file as its first
argument and can extract the relevant schema from there.
{% endblock %}
#### Generating test code
After you exported the database schema into a folder, you can generate old versions of your database class

View File

@ -2,6 +2,7 @@
- Add the support for `textEnum`.
- Adds the `case_from_dart_to_sql` option with the possible values: `preserve`, `camelCase`, `CONSTANT_CASE`, `snake_case`, `PascalCase`, `lowercase` and `UPPERCASE` (default: `snake_case`).
- `drift_dev schema dump` can now dump the schema of existing sqlite3 database files as well.
## 2.3.3

View File

@ -28,7 +28,6 @@ class DriftTableResolver extends LocalElementResolver<DiscoveredDriftTable> {
Table table;
final references = <DriftElement>{};
final stmt = discovered.sqlNode;
final helper = await resolver.driver.loadKnownTypes();
try {
final reader = SchemaFromCreateTable(
@ -75,7 +74,7 @@ class DriftTableResolver extends LocalElementResolver<DiscoveredDriftTable> {
DriftAnalysisError.inDriftFile(column.definition ?? stmt, msg)),
dartClass.classElement.thisType,
type == DriftSqlType.int ? EnumType.intEnum : EnumType.textEnum,
helper,
await resolver.driver.loadKnownTypes(),
);
}
}

View File

@ -1,11 +1,15 @@
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:args/command_runner.dart';
import 'package:collection/collection.dart';
import 'package:path/path.dart';
import 'package:sqlite3/sqlite3.dart';
import '../../../analysis/results/results.dart';
import '../../../services/schema/schema_files.dart';
import '../../../services/schema/sqlite_to_drift.dart';
import '../../cli.dart';
class DumpSchemaCommand extends Command {
@ -35,11 +39,68 @@ class DumpSchemaCommand extends Command {
usageException('Expected input and output files');
}
final absolute = File(rest[0]).absolute;
final _AnalyzedDatabase result;
if (await absolute.isSqlite3File) {
result = await _readElementsFromDatabase(absolute);
} else {
result = await _readElementsFromSource(absolute);
}
final writer =
SchemaWriter(result.elements, options: cli.project.moorOptions);
var target = rest[1];
// This command is most commonly used to write into
// `<dir>/drift_schema_vx.json`. When we get a directory as a second arg,
// try to infer the file name.
if (await FileSystemEntity.isDirectory(target) ||
!target.endsWith('.json')) {
final version = result.schemaVersion;
if (version == null) {
// Couldn't read schema from database, so fail.
usageException(
'Target is a directory and the schema version could not be read from '
'the database class. Please use a full filename (e.g. '
'`$target/drift_schema_v3.json`)',
);
}
target = join(target, 'drift_schema_v$version.json');
}
final file = File(target).absolute;
final parent = file.parent;
if (!await parent.exists()) {
await parent.create(recursive: true);
}
await file.writeAsString(json.encode(writer.createSchemaJson()));
print('Wrote to $target');
}
/// Reads available drift elements from an existing sqlite database file.
Future<_AnalyzedDatabase> _readElementsFromDatabase(File database) async {
final opened = sqlite3.open(database.path);
try {
final elements = await extractDriftElementsFromDatabase(opened);
final userVersion = opened.select('pragma user_version').single[0] as int;
return _AnalyzedDatabase(elements, userVersion);
} finally {
opened.dispose();
}
}
/// Extracts available drift elements from a [dart] source file defining a
/// drift database class.
Future<_AnalyzedDatabase> _readElementsFromSource(File dart) async {
final driver = await cli.createMoorDriver();
final absolute = File(rest[0]).absolute.path;
final input =
await driver.driver.fullyAnalyze(driver.uriFromPath(absolute));
await driver.driver.fullyAnalyze(driver.uriFromPath(dart.path));
if (!input.isFullyAnalyzed) {
cli.exit('Unexpected error: The input file could not be analyzed');
@ -56,36 +117,37 @@ class DumpSchemaCommand extends Command {
final databaseElement = databases.single;
final db = result.resolvedDatabases[databaseElement.id]!;
final writer =
SchemaWriter(db.availableElements, options: cli.project.moorOptions);
var target = rest[1];
// This command is most commonly used to write into
// `<dir>/drift_schema_vx.json`. When we get a directory as a second arg,
// try to infer the file name.
if (await FileSystemEntity.isDirectory(target) ||
!target.endsWith('.json')) {
final version = databaseElement.schemaVersion;
if (version == null) {
// Couldn't read schema from database, so fail.
usageException(
'Target is a directory and the schema version could not be read from '
'the database class. Please use a full filename (e.g. '
'`$target/drift_schema_v3.json`)',
);
}
target = join(target, 'drift_schema_v$version.json');
}
final file = File(target);
final parent = file.parent;
if (!await parent.exists()) {
await parent.create(recursive: true);
}
await File(target).writeAsString(json.encode(writer.createSchemaJson()));
print('Wrote to $target');
return _AnalyzedDatabase(
db.availableElements, databaseElement.schemaVersion);
}
}
extension on File {
static final _headerStart = ascii.encode('SQLite format 3\u0000');
/// Checks whether the file is probably a sqlite3 database file by looking at
/// the initial bytes of the expected header.
Future<bool> get isSqlite3File async {
final opened = await open();
try {
final bytes = Uint8List(_headerStart.length);
final bytesRead = await opened.readInto(bytes);
if (bytesRead < bytes.length) {
return false;
}
return const ListEquality<int>().equals(_headerStart, bytes);
} finally {
await opened.close();
}
}
}
class _AnalyzedDatabase {
final List<DriftElement> elements;
final int? schemaVersion;
_AnalyzedDatabase(this.elements, this.schemaVersion);
}

View File

@ -126,6 +126,7 @@ class SchemaWriter {
'without_rowid': table.withoutRowId,
if (table.overrideTableConstraints != null)
'constraints': table.overrideTableConstraints,
if (table.strict) 'strict': true,
if (primaryKeyFromTableConstraint != null)
'explicit_pk': [
...primaryKeyFromTableConstraint.primaryKey.map((c) => c.nameInSql)
@ -360,6 +361,7 @@ class SchemaReader {
columns: columns,
baseDartName: pascalCase,
fixedEntityInfoName: pascalCase,
strict: content['strict'] == true,
nameOfRowClass: '${pascalCase}Data',
writeDefaultConstraints: content['was_declared_in_moor'] != true,
withoutRowId: withoutRowId,

View File

@ -0,0 +1,85 @@
import 'package:analyzer/dart/element/element.dart';
import 'package:drift_dev/src/analysis/options.dart';
import 'package:logging/logging.dart';
import 'package:sqlite3/common.dart';
import 'package:sqlparser/sqlparser.dart';
import '../../analysis/backend.dart';
import '../../analysis/driver/driver.dart';
import '../../analysis/results/results.dart';
/// Extracts drift elements from the schema of an existing database.
///
/// At the moment, this is used to generate database schema files for databases
/// (as an alternative to using static analysis to infer the expected schema).
/// In the future, this could also be a starting point for users with existing
/// databases wanting to migrate to drift.
Future<List<DriftElement>> extractDriftElementsFromDatabase(
CommonDatabase database) async {
// Put everything from sqlite_schema into a fake drift file, analyze it.
final contents = database
.select('select * from sqlite_master')
.map((row) => row['sql'])
.whereType<String>()
.map((sql) => sql.endsWith(';') ? sql : '$sql;')
.join('\n');
final logger = Logger('extractDriftElementsFromDatabase');
final uri = Uri.parse('db.drift');
final backend = _SingleFileNoAnalyzerBackend(logger, contents, uri);
final driver = DriftAnalysisDriver(
backend,
DriftOptions.defaults(
sqliteAnalysisOptions: SqliteAnalysisOptions(
modules: SqlModule.values,
version: SqliteVersion.current,
),
),
);
final file = await driver.fullyAnalyze(uri);
return [
for (final entry in file.analysis.values)
if (entry.result != null) entry.result!
];
}
class _SingleFileNoAnalyzerBackend extends DriftBackend {
@override
final Logger log;
final String file;
final Uri uri;
_SingleFileNoAnalyzerBackend(this.log, this.file, this.uri);
Never _noAnalyzer() =>
throw UnsupportedError('Dart analyzer not available here');
@override
Future<Never> loadElementDeclaration(Element element) async {
_noAnalyzer();
}
@override
Future<String> readAsString(Uri uri) {
return Future.value(file);
}
@override
Future<LibraryElement> readDart(Uri uri) async {
_noAnalyzer();
}
@override
Future<Never> resolveExpression(
Uri context, String dartExpression, Iterable<String> imports) async {
_noAnalyzer();
}
@override
Uri resolveUri(Uri base, String uriString) {
return uri;
}
}

View File

@ -1,94 +1,42 @@
import 'dart:async';
import 'dart:io';
import 'dart:isolate';
import 'package:drift_dev/src/cli/cli.dart';
import 'package:meta/meta.dart';
import 'package:package_config/package_config_types.dart';
import 'package:path/path.dart' as p;
import 'package:test/scaffolding.dart';
import 'package:test/test.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;
@isTest
void _test(String desc, Function() body) {
test(desc, () {
return IOOverrides.runZoned(
body,
getCurrentDirectory: () => Directory('${d.sandbox}/app'),
);
});
import 'utils.dart';
extension on TestDriftProject {
Future<void> migrateToDrift() async {
await runDriftCli(['migrate']);
}
}
Future<void> _apply() {
return MoorCli().run(['migrate']);
}
Future<void> _setup(Iterable<d.Descriptor> lib,
{String? pubspec, Iterable<d.Descriptor>? additional}) async {
// Copy and patch moor_generator's package config instead of running `pub get`
// in each test.
final uri = await Isolate.packageConfig;
final config =
PackageConfig.parseBytes(await File.fromUri(uri!).readAsBytes(), uri);
final driftDevUrl =
config.packages.singleWhere((e) => e.name == 'drift_dev').root;
final moorUrl = driftDevUrl.resolve('../extras/assets/old_moor_package/');
final moorFlutterUrl =
driftDevUrl.resolve('../extras/assets/old_moor_flutter_package/');
final appUri = '${File(p.join(d.sandbox, 'app')).absolute.uri}/';
final newConfig = PackageConfig([
...config.packages,
Package('app', Uri.parse(appUri),
packageUriRoot: Uri.parse('${appUri}lib/')),
Package('moor', moorUrl, packageUriRoot: Uri.parse('${moorUrl}lib/')),
Package('moor_flutter', moorFlutterUrl,
packageUriRoot: Uri.parse('${moorFlutterUrl}lib/')),
]);
final configBuffer = StringBuffer();
PackageConfig.writeString(newConfig, configBuffer);
pubspec ??= '''
name: app
environment:
sdk: ^2.12.0
dependencies:
moor: ^4.4.0
dev_dependencies:
moor_generator: ^4.4.0
''';
await d.dir('app', [
Future<TestDriftProject> _setup2(Iterable<d.Descriptor> lib,
{String? pubspec, Iterable<d.Descriptor>? additional}) {
return TestDriftProject.create([
d.dir('lib', lib),
d.file('pubspec.yaml', pubspec),
d.dir('.dart_tool', [
d.file('package_config.json', configBuffer.toString()),
]),
if (pubspec != null) d.file('pubspec.yaml', pubspec),
...?additional,
]).create();
]);
}
void main() {
_test('renames moor files', () async {
await _setup([
test('renames moor files', () async {
final project = await _setup2([
d.file('a.moor', "import 'b.moor';"),
d.file('b.moor', 'CREATE TABLE foo (x TEXT);'),
]);
await _apply();
await project.migrateToDrift();
await d.dir('app/lib', [
await project.validate(d.dir('lib', [
d.file('a.drift', "import 'b.drift';"),
d.file('b.drift', 'CREATE TABLE foo (x TEXT);'),
]).validate();
]));
});
_test('patches moor imports', () async {
await _setup([
test('patches moor imports', () async {
final project = await _setup2([
d.file('a.dart', '''
import 'package:moor/moor.dart' as moor;
import 'package:moor/extensions/moor_ffi.dart';
@ -99,9 +47,9 @@ export 'package:moor/fFI.dart';
'''),
]);
await _apply();
await project.migrateToDrift();
await d.dir('app/lib', [
await project.validate(d.dir('lib', [
d.file('a.dart', '''
import 'package:drift/drift.dart' as moor;
import 'package:drift/extensions/native.dart';
@ -110,11 +58,11 @@ import 'package:drift/src/some/internal/file.dart';
export 'package:drift/web.dart';
export 'package:drift/native.dart';
'''),
]).validate();
]));
});
_test('updates identifier names', () async {
await _setup([
test('updates identifier names', () async {
final project = await _setup2([
d.file('a.dart', '''
import 'package:moor/moor.dart';
import 'package:moor/ffi.dart' as ffi;
@ -147,9 +95,9 @@ void main() {
'''),
]);
await _apply();
await project.migrateToDrift();
await d.dir('app/lib', [
await project.validate(d.dir('lib', [
d.file('a.dart', '''
import 'package:drift/drift.dart';
import 'package:drift/native.dart' as ffi;
@ -180,11 +128,11 @@ void main() {
}
}
'''),
]).validate();
]));
});
_test('patches include args from @UseMoor and @UseDao', () async {
await _setup([
test('patches include args from @UseMoor and @UseDao', () async {
final project = await _setup2([
d.file('a.dart', '''
import 'package:moor/moor.dart';
@ -196,9 +144,9 @@ class MyDao {}
'''),
]);
await _apply();
await project.migrateToDrift();
await d.dir('app/lib', [
await project.validate(d.dir('lib', [
d.file('a.dart', '''
import 'package:drift/drift.dart';
@ -208,11 +156,11 @@ class MyDatabase {}
@DriftAccessor(include: {'package:x/y.drift'})
class MyDao {}
'''),
]).validate();
]));
});
_test('patches `.moor.dart` part statements', () async {
await _setup([
test('patches `.moor.dart` part statements', () async {
final project = await _setup2([
d.file('a.dart', r'''
import 'package:moor/moor.dart';
@ -227,9 +175,9 @@ class FooDao with _$FooDaoMixin {}
'''),
]);
await _apply();
await project.migrateToDrift();
await d.dir('app/lib', [
await project.validate(d.dir('lib', [
d.file('a.dart', r'''
import 'package:drift/drift.dart';
@ -240,11 +188,11 @@ part 'a.drift.dart';
)
class FooDao with _$FooDaoMixin {}
'''),
]).validate();
]));
});
_test('updates pubspec.yaml', () async {
await _setup(const [], pubspec: '''
test('updates pubspec.yaml', () async {
final project = await _setup2(const [], pubspec: '''
name: app
environment:
@ -267,10 +215,9 @@ dependency_overrides:
version: ^1.2.3
''');
await _apply();
await project.migrateToDrift();
await d.dir('app', [
d.file('pubspec.yaml', '''
await project.validate(d.file('pubspec.yaml', '''
name: app
environment:
@ -291,12 +238,11 @@ dependency_overrides:
drift_dev:
hosted: foo
version: ^1.2.3
'''),
]).validate();
'''));
});
_test('transforms build configuration files', () async {
await _setup(
test('transforms build configuration files', () async {
final project = await _setup2(
const [],
additional: [
d.file('build.yaml', r'''
@ -320,10 +266,9 @@ targets:
],
);
await _apply();
await project.migrateToDrift();
await d.dir('app', [
d.file('build.yaml', r'''
await project.validate(d.file('build.yaml', r'''
targets:
$default:
builders:
@ -340,12 +285,11 @@ targets:
drift_dev|not_shared:
options:
another: option
''')
]).validate();
'''));
});
_test('transforms analysis option files', () async {
await _setup(
test('transforms analysis option files', () async {
final project = await _setup2(
const [],
additional: [
d.file('analysis_options.yaml', '''
@ -359,22 +303,20 @@ analyzer:
],
);
await _apply();
await project.migrateToDrift();
await d.dir('app', [
d.file('analysis_options.yaml', r'''
await project.validate(d.file('analysis_options.yaml', r'''
# a comment
analyzer:
plugins:
# comment 2
- drift # another
# another
''')
]).validate();
'''));
});
_test('transforms moor_flutter usages', () async {
await _setup(
test('transforms moor_flutter usages', () async {
final project = await _setup2(
[
d.file('a.dart', r'''
import 'package:moor_flutter/moor_flutter.dart';
@ -407,10 +349,9 @@ dev_dependencies:
''',
);
await _apply();
await project.migrateToDrift();
await d.dir(
'app',
await project.validateDir(
[
d.file('pubspec.yaml', '''
name: app
@ -442,6 +383,6 @@ QueryExecutor _executor() {
'''),
]),
],
).validate();
);
});
}

View File

@ -0,0 +1,139 @@
import 'dart:convert';
import 'package:sqlite3/sqlite3.dart';
import 'package:path/path.dart' as p;
import 'package:test/test.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;
import '../utils.dart';
void main() {
test('extracts schema json from database file', () async {
final project = await TestDriftProject.create();
sqlite3.open(p.join(project.root.path, 'test.db'))
..execute('CREATE TABLE users (id int primary key, name text) STRICT;')
..execute('CREATE VIEW names AS SELECT name FROM users;')
..execute('CREATE TRIGGER to_upper AFTER UPDATE ON users BEGIN '
' UPDATE users SET name = upper(new.name) where id = new.id;'
'END;')
..execute('CREATE INDEX idx ON users (name);')
..execute('pragma user_version = 1234;')
..dispose();
await project
.runDriftCli(['schema', 'dump', 'test.db', 'drift_migrations/']);
await project.validate(d.dir('drift_migrations', [
d.file(
'drift_schema_v1234.json',
isA<String>().having(
json.decode,
'parsed as json',
json.decode('''
{
"_meta": {
"description": "This file contains a serialized version of schema entities for drift.",
"version": "1.0.0"
},
"options": {
"store_date_time_values_as_text": false
},
"entities": [
{
"id": 0,
"references": [],
"type": "table",
"data": {
"name": "users",
"was_declared_in_moor": true,
"columns": [
{
"name": "id",
"getter_name": "id",
"moor_type": "ColumnType.integer",
"nullable": false,
"customConstraints": "PRIMARY KEY",
"default_dart": null,
"default_client_dart": null,
"dsl_features": [
"primary-key"
]
},
{
"name": "name",
"getter_name": "name",
"moor_type": "ColumnType.text",
"nullable": true,
"customConstraints": "",
"default_dart": null,
"default_client_dart": null,
"dsl_features": []
}
],
"is_virtual": false,
"without_rowid": false,
"constraints": [],
"strict": true
}
},
{
"id": 1,
"references": [
0
],
"type": "view",
"data": {
"name": "names",
"sql": "CREATE VIEW names AS SELECT name FROM users;",
"dart_data_name": "Name",
"dart_info_name": "Names",
"columns": [
{
"name": "name",
"getter_name": "name",
"moor_type": "ColumnType.text",
"nullable": true,
"customConstraints": null,
"default_dart": null,
"default_client_dart": null,
"dsl_features": []
}
]
}
},
{
"id": 2,
"references": [
0
],
"type": "trigger",
"data": {
"on": 0,
"references_in_body": [
0
],
"name": "to_upper",
"sql": "CREATE TRIGGER to_upper AFTER UPDATE ON users BEGIN UPDATE users SET name = upper(new.name) where id = new.id;END;"
}
},
{
"id": 3,
"references": [
0
],
"type": "index",
"data": {
"on": 0,
"name": "idx",
"sql": "CREATE INDEX idx ON users (name);"
}
}
]
}
'''),
),
),
]));
});
}

View File

@ -0,0 +1,83 @@
import 'dart:io';
import 'dart:isolate';
import 'package:drift_dev/src/cli/cli.dart';
import 'package:package_config/package_config.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;
import 'package:path/path.dart' as p;
class TestDriftProject {
final Directory root;
TestDriftProject(this.root);
Future<void> runDriftCli(Iterable<String> args) {
return IOOverrides.runZoned(
() => MoorCli().run(args),
getCurrentDirectory: () => root,
);
}
Future<void> validate(d.Descriptor descriptor) {
return descriptor.validate(root.path);
}
Future<void> validateDir(Iterable<d.Descriptor> descriptors) {
return Future.wait(descriptors.map(validate));
}
static Future<TestDriftProject> create([
Iterable<d.Descriptor> packageContent = const Iterable.empty(),
]) async {
final hasPubspec = packageContent.any((e) => e.name == 'pubspec.yaml');
final actualContents = [...packageContent];
final appRoot = p.join(d.sandbox, 'app');
if (!hasPubspec) {
actualContents.add(d.file('pubspec.yaml', '''
name: app
environment:
sdk: ^2.12.0
dependencies:
drift:
dev_dependencies:
drift_dev:
'''));
}
// Instead of running `pub get` for each test, we just copy the package
// config used by drift_dev over.
final uri = await Isolate.packageConfig;
final config =
PackageConfig.parseBytes(await File.fromUri(uri!).readAsBytes(), uri);
final driftDevUrl =
config.packages.singleWhere((e) => e.name == 'drift_dev').root;
final moorUrl = driftDevUrl.resolve('../extras/assets/old_moor_package/');
final moorFlutterUrl =
driftDevUrl.resolve('../extras/assets/old_moor_flutter_package/');
final appUri = '${File(appRoot).absolute.uri}/';
final newConfig = PackageConfig([
...config.packages,
Package('app', Uri.parse(appUri),
packageUriRoot: Uri.parse('${appUri}lib/')),
// Also include old moor packages to test migration from moor to drift
Package('moor', moorUrl, packageUriRoot: Uri.parse('${moorUrl}lib/')),
Package('moor_flutter', moorFlutterUrl,
packageUriRoot: Uri.parse('${moorFlutterUrl}lib/')),
]);
final configBuffer = StringBuffer();
PackageConfig.writeString(newConfig, configBuffer);
actualContents.add(d.dir('.dart_tool', [
d.file('package_config.json', configBuffer.toString()),
]));
await d.dir('app', actualContents).create();
return TestDriftProject(Directory(appRoot));
}
}