From 5d8040554f39ca83527e1616e938907530705cee Mon Sep 17 00:00:00 2001 From: Simon Binder Date: Sun, 12 Jan 2020 10:38:03 +0100 Subject: [PATCH] CLI tool to export a moor schema to json --- moor/lib/src/runtime/types/custom_type.dart | 2 + moor/lib/src/runtime/types/sql_types.dart | 1 + moor_generator/lib/src/analyzer/errors.dart | 4 +- .../analyzer/runner/steps/analyze_dart.dart | 7 + moor_generator/lib/src/analyzer/session.dart | 10 + moor_generator/lib/src/cli/cli.dart | 15 +- .../lib/src/cli/commands/schema.dart | 16 ++ .../lib/src/cli/commands/schema/dump.dart | 56 ++++ moor_generator/lib/src/model/table.dart | 9 + .../lib/src/services/schema/writer.dart | 122 +++++++++ .../src/utils/entity_reference_sorter.dart | 12 +- .../test/services/schema/writer_test.dart | 259 ++++++++++++++++++ sqlparser/lib/src/reader/parser/parser.dart | 5 + 13 files changed, 511 insertions(+), 7 deletions(-) create mode 100644 moor_generator/lib/src/cli/commands/schema.dart create mode 100644 moor_generator/lib/src/cli/commands/schema/dump.dart create mode 100644 moor_generator/lib/src/services/schema/writer.dart create mode 100644 moor_generator/test/services/schema/writer_test.dart diff --git a/moor/lib/src/runtime/types/custom_type.dart b/moor/lib/src/runtime/types/custom_type.dart index a0c639d6..87c61e27 100644 --- a/moor/lib/src/runtime/types/custom_type.dart +++ b/moor/lib/src/runtime/types/custom_type.dart @@ -5,6 +5,8 @@ part of 'sql_types.dart'; /// /// Moor currently supports [DateTime], [double], [int], [Uint8List], [bool] /// and [String] for [S]. +/// +/// Also see [ColumnBuilder.map] for details. abstract class TypeConverter { /// Empty constant constructor so that subclasses can have a constant /// constructor. diff --git a/moor/lib/src/runtime/types/sql_types.dart b/moor/lib/src/runtime/types/sql_types.dart index 6142a3a9..4685f429 100644 --- a/moor/lib/src/runtime/types/sql_types.dart +++ b/moor/lib/src/runtime/types/sql_types.dart @@ -1,6 +1,7 @@ import 'dart:typed_data'; import 'package:convert/convert.dart'; +import 'package:moor/moor.dart'; part 'custom_type.dart'; part 'type_system.dart'; diff --git a/moor_generator/lib/src/analyzer/errors.dart b/moor_generator/lib/src/analyzer/errors.dart index a707d365..25ab1772 100644 --- a/moor_generator/lib/src/analyzer/errors.dart +++ b/moor_generator/lib/src/analyzer/errors.dart @@ -22,7 +22,9 @@ class MoorError { @override String toString() { - return 'Error: $message'; + final builder = StringBuffer(); + writeDescription((msg, [_, __]) => builder.writeln(msg)); + return 'Error: $builder'; } void writeDescription(LogFunction log) { diff --git a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart index d39b731d..3be2e06b 100644 --- a/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart +++ b/moor_generator/lib/src/analyzer/runner/steps/analyze_dart.dart @@ -32,6 +32,13 @@ class AnalyzeDartStep extends AnalyzingStep { affectedElement: accessor.fromClass, message: msg.toString(), )); + } catch (e) { + // unknown error while sorting + reportError(ErrorInDartCode( + severity: Severity.warning, + affectedElement: accessor.fromClass, + message: 'Unknown error while sorting database entities: $e', + )); } final availableQueries = transitiveImports diff --git a/moor_generator/lib/src/analyzer/session.dart b/moor_generator/lib/src/analyzer/session.dart index 125afae0..350586f1 100644 --- a/moor_generator/lib/src/analyzer/session.dart +++ b/moor_generator/lib/src/analyzer/session.dart @@ -1,5 +1,6 @@ import 'dart:async'; +import 'package:moor_generator/src/analyzer/errors.dart'; import 'package:moor_generator/src/analyzer/runner/file_graph.dart'; import 'package:moor_generator/src/analyzer/runner/task.dart'; import 'package:moor_generator/src/backends/backend.dart'; @@ -77,6 +78,15 @@ class MoorSession { return Task(this, _uriToFile(backend.entrypoint), backend); } + /// Finds all current errors in the [file] and transitive imports thereof. + Iterable errorsInFileAndImports(FoundFile file) { + final targetFiles = [file, ...fileGraph.crawl(file)]; + + return targetFiles.fold(const Iterable.empty(), (errors, file) { + return errors.followedBy(file.errors.errors); + }); + } + /// A stream emitting files whenever they were included in a completed task. Stream completedFiles() { return completedTasks.expand((task) => task.analyzedFiles); diff --git a/moor_generator/lib/src/cli/cli.dart b/moor_generator/lib/src/cli/cli.dart index 2730fe96..0c21518a 100644 --- a/moor_generator/lib/src/cli/cli.dart +++ b/moor_generator/lib/src/cli/cli.dart @@ -10,6 +10,7 @@ import 'package:moor_generator/src/cli/project.dart'; import 'commands/debug_plugin.dart'; import 'commands/identify_databases.dart'; +import 'commands/schema.dart'; import 'logging.dart'; Future run(List args) { @@ -37,9 +38,11 @@ class MoorCli { _runner = CommandRunner( 'pub run moor_generator', 'CLI utilities for the moor package, currently in an experimental state.', + usageLineLength: 80, ) ..addCommand(IdentifyDatabases(this)) - ..addCommand(DebugPluginCommand(this)); + ..addCommand(DebugPluginCommand(this)) + ..addCommand(SchemaCommand(this)); _runner.argParser .addFlag('verbose', abbr: 'v', defaultsTo: false, negatable: false); @@ -68,6 +71,10 @@ class MoorCli { await _runner.runCommand(results); } + + void exit(String message) { + throw FatalToolError(message); + } } abstract class MoorCommand extends Command { @@ -75,3 +82,9 @@ abstract class MoorCommand extends Command { MoorCommand(this.cli); } + +class FatalToolError implements Exception { + final String message; + + FatalToolError(this.message); +} diff --git a/moor_generator/lib/src/cli/commands/schema.dart b/moor_generator/lib/src/cli/commands/schema.dart new file mode 100644 index 00000000..d4de8c6f --- /dev/null +++ b/moor_generator/lib/src/cli/commands/schema.dart @@ -0,0 +1,16 @@ +import 'package:args/command_runner.dart'; +import 'package:moor_generator/src/cli/commands/schema/dump.dart'; + +import '../cli.dart'; + +class SchemaCommand extends Command { + @override + String get description => 'Inspect or manage the schema of a moor database'; + + @override + String get name => 'schema'; + + SchemaCommand(MoorCli cli) { + addSubcommand(DumpSchemaCommand(cli)); + } +} diff --git a/moor_generator/lib/src/cli/commands/schema/dump.dart b/moor_generator/lib/src/cli/commands/schema/dump.dart new file mode 100644 index 00000000..bd26e2f6 --- /dev/null +++ b/moor_generator/lib/src/cli/commands/schema/dump.dart @@ -0,0 +1,56 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/services/schema/writer.dart'; + +import '../../cli.dart'; + +class DumpSchemaCommand extends Command { + @override + String get description => 'Export the entire table structure into a file'; + + @override + String get name => 'dump'; + + @override + String get invocation { + return '${runner.executableName} schema dump [arguments] '; + } + + final MoorCli cli; + + DumpSchemaCommand(this.cli) { + argParser.addSeparator("It's recommended to run this commend from the " + 'directory containing your pubspec.yaml so that compiler options ' + 'are respected.'); + } + + @override + Future run() async { + final rest = argResults.rest; + if (rest.length != 2) { + usageException('Expected input and output files'); + } + + final driver = await cli.createMoorDriver(); + + final absolute = File(rest[0]).absolute.path; + final input = await driver.waitFileParsed(absolute); + + if (!input.isAnalyzed) { + cli.exit('Unexpected error: The input file could not be analyzed'); + } + + final result = input.currentResult; + if (result is! ParsedDartFile) { + cli.exit('Input file is not a Dart file'); + } + + final db = (result as ParsedDartFile).declaredDatabases.single; + final writer = SchemaWriter(db); + + await File(rest[1]).writeAsString(json.encode(writer.createSchemaJson())); + } +} diff --git a/moor_generator/lib/src/model/table.dart b/moor_generator/lib/src/model/table.dart index 07d61cd5..e59da462 100644 --- a/moor_generator/lib/src/model/table.dart +++ b/moor_generator/lib/src/model/table.dart @@ -100,6 +100,15 @@ class MoorTable implements MoorSchemaEntity { return node is CreateVirtualTableStatement; } + /// If this table [isVirtualTable], returns the `CREATE VIRTUAL TABLE` + /// statement to create this table. Otherwise returns null. + String get createVirtual { + if (!isVirtualTable) return null; + + final node = (declaration as MoorTableDeclaration).node; + return (node as CreateVirtualTableStatement).span.text; + } + MoorTable({ this.fromClass, this.columns, diff --git a/moor_generator/lib/src/services/schema/writer.dart b/moor_generator/lib/src/services/schema/writer.dart new file mode 100644 index 00000000..772b2ce8 --- /dev/null +++ b/moor_generator/lib/src/services/schema/writer.dart @@ -0,0 +1,122 @@ +import 'package:moor_generator/moor_generator.dart'; + +const _infoVersion = '0.1.0-dev-preview'; + +/// Utilities to transform moor schema entities to json. +class SchemaWriter { + /// The parsed and resolved database for which the schema should be written. + final Database db; + + final Map _entityIds = {}; + int _maxId = 0; + + SchemaWriter(this.db); + + int _idOf(MoorSchemaEntity entity) { + return _entityIds.putIfAbsent(entity, () => _maxId++); + } + + Map createSchemaJson() { + return { + '_meta': { + 'description': 'This file contains a serialized version of schema ' + 'entities for moor.', + 'version': _infoVersion, + }, + 'entities': [ + for (final entity in db.entities) _entityToJson(entity), + ], + }; + } + + Map _entityToJson(MoorSchemaEntity entity) { + String type; + Map data; + + if (entity is MoorTable) { + type = 'table'; + data = _tableData(entity); + } else if (entity is MoorTrigger) { + type = 'trigger'; + data = { + 'on': _idOf(entity.on), + 'refences_in_body': [ + for (final ref in entity.bodyReferences) _idOf(ref), + ], + 'name': entity.displayName, + 'sql': entity.create, + }; + } else if (entity is MoorIndex) { + type = 'index'; + data = { + 'on': _idOf(entity.table), + 'name': entity.name, + 'sql': entity.createStmt, + }; + } else if (entity is SpecialQuery) { + type = 'special-query'; + data = { + 'scenario': 'create', + 'sql': entity.sql, + }; + } + + return { + 'id': _idOf(entity), + 'references': [ + for (final reference in entity.references) _idOf(reference), + ], + 'type': type, + 'data': data, + }; + } + + Map _tableData(MoorTable table) { + return { + 'name': table.sqlName, + 'was_declared_in_moor': table.isFromSql, + 'columns': [for (final column in table.columns) _columnData(column)], + 'is_virtual': table.isVirtualTable, + if (table.isVirtualTable) 'create_virtual_stmt': table.createVirtual, + if (table.overrideWithoutRowId != null) + 'without_rowid': table.overrideWithoutRowId, + if (table.overrideTableConstraints != null) + 'constraints': table.overrideTableConstraints, + if (table.primaryKey != null) + 'explicit_pk': [...table.primaryKey.map((c) => c.name.name)] + }; + } + + Map _columnData(MoorColumn column) { + return { + 'name': column.name.name, + 'moor_type': column.type.toString(), + 'nullable': column.nullable, + 'customConstraints': column.customConstraints, + 'default_dart': column.defaultArgument, + 'default_client_dart': column.clientDefaultCode, + 'dsl_features': [...column.features.map(_dslFeatureData)], + if (column.typeConverter != null) + 'type_converter': { + 'dart_expr': column.typeConverter.expression.toSource(), + 'dart_type_name': column.typeConverter.mappedType.displayName, + } + }; + } + + dynamic _dslFeatureData(ColumnFeature feature) { + if (feature is AutoIncrement) { + return 'auto-increment'; + } else if (feature is PrimaryKey) { + return 'primary-key'; + } else if (feature is LimitingTextLength) { + return { + 'allowed-lengths': { + 'min': feature.minLength, + 'max': feature.maxLength, + }, + }; + } + return 'unknown'; + } +} diff --git a/moor_generator/lib/src/utils/entity_reference_sorter.dart b/moor_generator/lib/src/utils/entity_reference_sorter.dart index f3de3624..a267cff5 100644 --- a/moor_generator/lib/src/utils/entity_reference_sorter.dart +++ b/moor_generator/lib/src/utils/entity_reference_sorter.dart @@ -21,21 +21,23 @@ List sortEntitiesTopologically( return run.result; } -void _visit(MoorSchemaEntity table, _SortRun run) { - for (final reference in table.references) { +void _visit(MoorSchemaEntity entity, _SortRun run) { + for (final reference in entity.references) { + assert(reference != null, '$entity had a null reference'); + if (run.result.contains(reference)) { // already handled, nothing to do } else if (run.previous.containsKey(reference)) { // that's a circular reference, report - run.throwCircularException(table, reference); + run.throwCircularException(entity, reference); } else { - run.previous[reference] = table; + run.previous[reference] = entity; _visit(reference, run); } } // now that everything this table references is written, add the table itself - run.result.add(table); + run.result.add(entity); } class _SortRun { diff --git a/moor_generator/test/services/schema/writer_test.dart b/moor_generator/test/services/schema/writer_test.dart new file mode 100644 index 00000000..6dde26ed --- /dev/null +++ b/moor_generator/test/services/schema/writer_test.dart @@ -0,0 +1,259 @@ +import 'dart:convert'; + +import 'package:moor_generator/src/analyzer/runner/results.dart'; +import 'package:moor_generator/src/services/schema/writer.dart'; +import 'package:test/test.dart'; + +import '../../analyzer/utils.dart'; + +void main() { + test('writer integration test', () async { + final state = TestState.withContent({ + 'foo|lib/a.moor': ''' +import 'main.dart'; + +CREATE TABLE "groups" ( + id INT NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + + UNIQUE(name) +); + +CREATE TABLE group_members ( + "group" INT NOT NULL REFERENCES "groups"(id), + user INT NOT NULL REFERENCES users(id), + is_admin BOOLEAN NOT NULL DEFAULT FALSE, + + PRIMARY KEY ("group", user) ON CONFLICT REPLACE +); + +CREATE TRIGGER delete_empty_groups AFTER DELETE ON group_members BEGIN + DELETE FROM "groups" g + WHERE NOT EXISTS (SELECT * FROM group_members WHERE "group" = g.id); +END; + +CREATE INDEX groups_name ON "groups"(name); + ''', + 'foo|lib/main.dart': ''' +import 'package:moor/moor.dart'; + +class Users extends Table { + IntColumn get id => integer().autoIncrement()(); + TextColumn get name => text()(); + TextColumn get settings => text().map(const SettingsConverter())(); +} + +class Settings {} + +class SettingsConverter extends TypeConverter { + const SettingsConverter(); + + String mapToSql(Settings s) => ''; + Settings mapToDart(String db) => Settings(); +} + +@UseMoor(include: {'a.moor'}, tables: [Users]) +class Database {} + ''', + }); + + final file = await state.analyze('package:foo/main.dart'); + expect(state.session.errorsInFileAndImports(file), isEmpty); + + final result = file.currentResult as ParsedDartFile; + final db = result.declaredDatabases.single; + + final schemaJson = SchemaWriter(db).createSchemaJson(); + expect(schemaJson, json.decode(expected)); + }); +} + +const expected = r''' +{ + "_meta":{ + "description":"This file contains a serialized version of schema entities for moor.", + "version":"0.1.0-dev-preview" + }, + "entities":[ + { + "id":0, + "references":[ + + ], + "type":"table", + "data":{ + "name":"groups", + "was_declared_in_moor":true, + "columns":[ + { + "name":"id", + "moor_type":"ColumnType.integer", + "nullable":false, + "customConstraints":"NOT NULL PRIMARY KEY AUTOINCREMENT", + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + "primary-key", + "auto-increment" + ] + }, + { + "name":"name", + "moor_type":"ColumnType.text", + "nullable":false, + "customConstraints":"NOT NULL", + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + + ] + } + ], + "is_virtual":false, + "constraints":[ + "UNIQUE(name)" + ], + "explicit_pk":[ + "id" + ] + } + }, + { + "id":1, + "references":[ + + ], + "type":"table", + "data":{ + "name":"users", + "was_declared_in_moor":false, + "columns":[ + { + "name":"id", + "moor_type":"ColumnType.integer", + "nullable":false, + "customConstraints":null, + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + "auto-increment", + "primary-key" + ] + }, + { + "name":"name", + "moor_type":"ColumnType.text", + "nullable":false, + "customConstraints":null, + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + + ] + }, + { + "name":"settings", + "moor_type":"ColumnType.text", + "nullable":false, + "customConstraints":null, + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + + ], + "type_converter":{ + "dart_expr":"const SettingsConverter()", + "dart_type_name":"Settings" + } + } + ], + "is_virtual":false + } + }, + { + "id":2, + "references":[ + 0, + 1 + ], + "type":"table", + "data":{ + "name":"group_members", + "was_declared_in_moor":true, + "columns":[ + { + "name":"group", + "moor_type":"ColumnType.integer", + "nullable":false, + "customConstraints":"NOT NULL REFERENCES \"groups\"(id)", + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + + ] + }, + { + "name":"user", + "moor_type":"ColumnType.integer", + "nullable":false, + "customConstraints":"NOT NULL REFERENCES users(id)", + "default_dart":null, + "default_client_dart":null, + "dsl_features":[ + + ] + }, + { + "name":"is_admin", + "moor_type":"ColumnType.boolean", + "nullable":false, + "customConstraints":"NOT NULL DEFAULT FALSE", + "default_dart":"const CustomExpression('FALSE')", + "default_client_dart":null, + "dsl_features":[ + + ] + } + ], + "is_virtual":false, + "constraints":[ + "PRIMARY KEY (\"group\", user) ON CONFLICT REPLACE" + ], + "explicit_pk":[ + "group", + "user" + ] + } + }, + { + "id":3, + "references":[ + 2, + 0 + ], + "type":"trigger", + "data":{ + "on":2, + "refences_in_body":[ + 0, + 2 + ], + "name":"delete_empty_groups", + "sql":"CREATE TRIGGER delete_empty_groups AFTER DELETE ON group_members BEGIN\n DELETE FROM \"groups\" g\n WHERE NOT EXISTS (SELECT * FROM group_members WHERE \"group\" = g.id);\nEND;" + } + }, + { + "id":4, + "references":[ + 0 + ], + "type":"index", + "data":{ + "on":0, + "name":"groups_name", + "sql":"CREATE INDEX groups_name ON \"groups\"(name);" + } + } + ] +} +'''; diff --git a/sqlparser/lib/src/reader/parser/parser.dart b/sqlparser/lib/src/reader/parser/parser.dart index ae2f8d04..f3afb192 100644 --- a/sqlparser/lib/src/reader/parser/parser.dart +++ b/sqlparser/lib/src/reader/parser/parser.dart @@ -157,6 +157,11 @@ abstract class ParserBase { if (next is KeywordToken && (next.canConvertToIdentifier() || lenient)) { return (_advance() as KeywordToken).convertToIdentifier(); } + + if (next is KeywordToken) { + message = '$message (got keyword ${reverseKeywords[next.type]})'; + } + return _consume(TokenType.identifier, message) as IdentifierToken; }