Start some work on the analyzer plugin

Of course it's not working at all

analysis_options.yaml

@@ -28,7 +28,6 @@ linter:
     - await_only_futures
     - camel_case_types
     - cancel_subscriptions
-    - cascade_invocations
     - comment_references
     - constant_identifier_names
     - curly_braces_in_flow_control_structures

extras/integration_tests/tests/analysis_options.yaml

@@ -0,0 +1,3 @@
+analyzer:
+  plugins:
+   - moor

extras/integration_tests/tests/lib/database/test.moor

@@ -0,0 +1,3 @@
+CREATE TABLE test (
+    id INT NOT NULL PRIMARY AUTOINCREMENT
+)

moor/tools/analyzer_plugin/pubspec.yaml

@@ -3,8 +3,10 @@ version: 1.0.0
 description: This pubspec is a part of moor and determines the version of the moor analyzer to load
 
 dependencies:
-  moor_generator: ^1.6.0
-
-dependency_overrides:
   moor_generator:
-    path: ../../../moor_generator
+
+#dependency_overrides:
+#  moor_generator:
+#    path: /home/simon/IdeaProjects/moor/moor_generator
+#  sqlparser:
+#    path: /home/simon/IdeaProjects/moor/sqlparser

moor_generator/lib/src/parser/moor/moor_analyzer.dart

@@ -11,7 +11,9 @@ class MoorAnalyzer {
   MoorAnalyzer(this.content);
 
   Future<MoorParsingResult> analyze() {
-    final results = SqlEngine().parseMultiple(content);
+    final engine = SqlEngine();
+    final tokens = engine.tokenize(content);
+    final results = SqlEngine().parseMultiple(tokens, content);
 
     final createdTables = <CreateTable>[];
     final errors = <MoorParsingError>[];

moor_generator/lib/src/plugin/analyzer/highlights/request.dart

@@ -0,0 +1,14 @@
+import 'package:analyzer/file_system/file_system.dart';
+import 'package:analyzer_plugin/utilities/highlights/highlights.dart';
+
+import '../results.dart';
+
+class MoorHighlightingRequest extends HighlightsRequest {
+  @override
+  final String path;
+  @override
+  final ResourceProvider resourceProvider;
+  final MoorAnalysisResults parsedFile;
+
+  MoorHighlightingRequest(this.parsedFile, this.path, this.resourceProvider);
+}

moor_generator/lib/src/plugin/analyzer/highlights/sql_highlighter.dart

@@ -0,0 +1,82 @@
+import 'package:analyzer_plugin/protocol/protocol_common.dart';
+import 'package:analyzer_plugin/utilities/highlights/highlights.dart';
+import 'package:moor_generator/src/plugin/analyzer/highlights/request.dart';
+import 'package:sqlparser/sqlparser.dart';
+
+const _notBuiltIn = {
+  TokenType.numberLiteral,
+  TokenType.stringLiteral,
+  TokenType.identifier,
+  TokenType.leftParen,
+  TokenType.rightParen,
+  TokenType.comma,
+  TokenType.star,
+  TokenType.less,
+  TokenType.lessEqual,
+  TokenType.lessMore,
+  TokenType.equal,
+  TokenType.more,
+  TokenType.moreEqual,
+  TokenType.shiftRight,
+  TokenType.shiftLeft,
+  TokenType.exclamationEqual,
+  TokenType.plus,
+  TokenType.minus,
+};
+
+class SqlHighlighter implements HighlightsContributor {
+  const SqlHighlighter();
+
+  @override
+  void computeHighlights(
+      HighlightsRequest request, HighlightsCollector collector) {
+    if (request is! MoorHighlightingRequest) {
+      return;
+    }
+
+    final typedRequest = request as MoorHighlightingRequest;
+    final visitor = _HighlightingVisitor(collector);
+
+    for (var stmt in typedRequest.parsedFile.statements) {
+      stmt.accept(visitor);
+    }
+
+    for (var token in typedRequest.parsedFile.sqlTokens) {
+      if (!_notBuiltIn.contains(token.type)) {
+        final start = token.span.start.offset;
+        final length = token.span.length;
+        collector.addRegion(start, length, HighlightRegionType.BUILT_IN);
+      }
+    }
+  }
+}
+
+class _HighlightingVisitor extends RecursiveVisitor<void> {
+  final HighlightsCollector collector;
+
+  _HighlightingVisitor(this.collector);
+
+  void _contribute(AstNode node, HighlightRegionType type) {
+    final offset = node.firstPosition;
+    final length = node.lastPosition - offset;
+    collector.addRegion(offset, length, type);
+  }
+
+  @override
+  void visitReference(Reference e) {
+    _contribute(e, HighlightRegionType.INSTANCE_FIELD_REFERENCE);
+  }
+
+  @override
+  void visitLiteral(Literal e) {
+    if (e is NullLiteral) {
+      _contribute(e, HighlightRegionType.BUILT_IN);
+    } else if (e is NumericLiteral) {
+      _contribute(e, HighlightRegionType.LITERAL_INTEGER);
+    } else if (e is StringLiteral) {
+      _contribute(e, HighlightRegionType.LITERAL_STRING);
+    } else if (e is BooleanLiteral) {
+      _contribute(e, HighlightRegionType.LITERAL_BOOLEAN);
+    }
+  }
+}

moor_generator/lib/src/plugin/analyzer/moor_analyzer.dart

@@ -0,0 +1,15 @@
+import 'package:analyzer/file_system/file_system.dart';
+import 'package:moor_generator/src/plugin/analyzer/results.dart';
+import 'package:sqlparser/sqlparser.dart';
+
+class MoorAnalyzer {
+  Future<MoorAnalysisResults> analyze(File file) async {
+    final content = file.readAsStringSync();
+    final sqlEngine = SqlEngine();
+
+    final tokens = sqlEngine.tokenize(content);
+    final stmts = sqlEngine.parseMultiple(tokens, content);
+
+    return MoorAnalysisResults(stmts.map((r) => r.rootNode).toList(), tokens);
+  }
+}

moor_generator/lib/src/plugin/analyzer/results.dart

@@ -0,0 +1,8 @@
+import 'package:sqlparser/sqlparser.dart';
+
+class MoorAnalysisResults {
+  final List<AstNode> statements;
+  final List<Token> sqlTokens;
+
+  MoorAnalysisResults(this.statements, this.sqlTokens);
+}

moor_generator/lib/src/plugin/driver.dart

@@ -1,40 +1,79 @@
 // ignore_for_file: implementation_imports
+import 'dart:async';
+
+import 'package:analyzer/file_system/file_system.dart';
 import 'package:analyzer/src/dart/analysis/driver.dart';
+import 'package:moor_generator/src/plugin/state/file_tracker.dart';
+
+import 'analyzer/moor_analyzer.dart';
+import 'analyzer/results.dart';
 
 class MoorDriver implements AnalysisDriverGeneric {
-  final _addedFiles = <String>{};
+  final FileTracker _tracker;
+  final AnalysisDriverScheduler _scheduler;
+  final MoorAnalyzer _analyzer;
+  final ResourceProvider _resources;
+
+  MoorDriver(this._tracker, this._scheduler, this._analyzer, this._resources) {
+    _scheduler.add(this);
+  }
 
   bool _ownsFile(String path) => path.endsWith('.moor');
 
   @override
   void addFile(String path) {
     if (_ownsFile(path)) {
-      _addedFiles.add(path);
+      _tracker.addFile(path);
-      handleFileChanged(path);
     }
   }
 
   @override
-  void dispose() {}
+  void dispose() {
+    _scheduler.remove(this);
+  }
 
   void handleFileChanged(String path) {
-    if (_ownsFile(path)) {}
+    if (_ownsFile(path)) {
+      _tracker.handleContentChanged(path);
+      _scheduler.notify(this);
+    }
   }
 
   @override
-  bool get hasFilesToAnalyze => null;
+  bool get hasFilesToAnalyze => _tracker.hasWork;
 
   @override
-  Future<void> performWork() {
+  Future<void> performWork() async {
-    // TODO: implement performWork
+    final completer = Completer();
-    return null;
+
+    if (_tracker.hasWork) {
+      _tracker.work((path) {
+        try {
+          return _resolveMoorFile(path);
+        } finally {
+          completer.complete();
+        }
+      });
+
+      await completer.future;
+    }
+  }
+
+  Future<MoorAnalysisResults> _resolveMoorFile(String path) {
+    return _analyzer.analyze(_resources.getFile(path));
   }
 
   @override
   set priorityFiles(List<String> priorityPaths) {
-    // We don't support this ATM
+    _tracker.setPriorityFiles(priorityPaths);
   }
 
   @override
+  // todo ask the tracker about the top-priority file.
   AnalysisDriverPriority get workPriority => AnalysisDriverPriority.general;
+
+  Future<MoorAnalysisResults> parseMoorFile(String path) {
+    _scheduler.notify(this);
+    return _tracker.results(path);
+  }
 }

moor_generator/lib/src/plugin/plugin.dart

@@ -1,24 +1,66 @@
 import 'package:analyzer/file_system/file_system.dart';
-// ignore: implementation_imports
+import 'package:analyzer_plugin/plugin/highlights_mixin.dart';
-import 'package:analyzer/src/dart/analysis/driver.dart';
 import 'package:analyzer_plugin/plugin/plugin.dart';
+import 'package:analyzer_plugin/protocol/protocol.dart';
 import 'package:analyzer_plugin/protocol/protocol_generated.dart';
+import 'package:analyzer_plugin/utilities/highlights/highlights.dart';
+import 'package:moor_generator/src/plugin/state/file_tracker.dart';
 
-class MoorPlugin extends ServerPlugin {
+import 'analyzer/highlights/request.dart';
+import 'analyzer/highlights/sql_highlighter.dart';
+import 'analyzer/moor_analyzer.dart';
+import 'driver.dart';
+
+class MoorPlugin extends ServerPlugin with HighlightsMixin {
   MoorPlugin(ResourceProvider provider) : super(provider);
 
   @override
-  final List<String> fileGlobsToAnalyze = const ['**/*.moor'];
+  final List<String> fileGlobsToAnalyze = const ['*.moor'];
   @override
   final String name = 'Moor plugin';
   @override
-  final String version = '0.0.1';
+  // docs say that this should a version of _this_ plugin, but they lie. this
+  // version will be used to determine compatibility with the analyzer
+  final String version = '2.0.0-alpha.0';
   @override
   final String contactInfo =
       'Create an issue at https://github.com/simolus3/moor/';
 
   @override
-  AnalysisDriverGeneric createAnalysisDriver(ContextRoot contextRoot) {
+  MoorDriver createAnalysisDriver(ContextRoot contextRoot) {
-    return null;
+    final tracker = FileTracker();
+    final analyzer = MoorAnalyzer();
+    return MoorDriver(
+        tracker, analysisDriverScheduler, analyzer, resourceProvider);
+  }
+
+  @override
+  void contentChanged(String path) {
+    _moorDriverForPath(path)?.handleFileChanged(path);
+  }
+
+  MoorDriver _moorDriverForPath(String path) {
+    final driver = super.driverForPath(path);
+
+    if (driver is! MoorDriver) return null;
+    return driver as MoorDriver;
+  }
+
+  @override
+  List<HighlightsContributor> getHighlightsContributors(String path) {
+    return const [SqlHighlighter()];
+  }
+
+  @override
+  Future<HighlightsRequest> getHighlightsRequest(String path) async {
+    final driver = _moorDriverForPath(path);
+    if (driver == null) {
+      throw RequestFailure(
+          RequestErrorFactory.pluginError('Not driver set for path', null));
+    }
+
+    final parsed = await driver.parseMoorFile(path);
+
+    return MoorHighlightingRequest(parsed, path, resourceProvider);
   }
 }

moor_generator/lib/src/plugin/state/file_tracker.dart

@@ -1,5 +1,128 @@
-/// Keeps track of files that need to be analyzed by the moor generator.
+import 'dart:async';
-class FileTracker {}
 
-/// A `.moor` file added to the plugin.
+import 'package:collection/collection.dart';
-class _TrackedFile {}
+import 'package:moor_generator/src/plugin/analyzer/results.dart';
+
+/// Keeps track of files that need to be analyzed by the moor plugin.
+class FileTracker {
+  PriorityQueue<TrackedFile> _pendingWork;
+  final Map<String, TrackedFile> _trackedFiles = {};
+  final Set<TrackedFile> _currentPriority = {};
+
+  FileTracker() {
+    _pendingWork = PriorityQueue(_compareByPriority);
+  }
+
+  int _compareByPriority(TrackedFile a, TrackedFile b) {
+    final aPriority = a.currentPriority?.index ?? 0;
+    final bPriority = b.currentPriority?.index ?? 0;
+    return aPriority.compareTo(bPriority);
+  }
+
+  void _updateFile(TrackedFile file, Function(TrackedFile) update) {
+    _pendingWork.remove(file);
+    update(file);
+    _pendingWork.add(file);
+  }
+
+  void _putInQueue(TrackedFile file) {
+    _updateFile(file, (f) {
+      // no action needed, insert with current priority.
+    });
+  }
+
+  bool get hasWork => _pendingWork.isNotEmpty;
+
+  TrackedFile addFile(String path) {
+    return _trackedFiles.putIfAbsent(path, () {
+      final tracked = TrackedFile(path);
+      _pendingWork.add(tracked);
+      return tracked;
+    });
+  }
+
+  void handleContentChanged(String path) {
+    _putInQueue(addFile(path));
+  }
+
+  void setPriorityFiles(List<String> priority) {
+    // remove prioritized flag from existing files
+    for (var file in _currentPriority) {
+      _updateFile(file, (f) => f._prioritized = false);
+    }
+    _currentPriority
+      ..clear()
+      ..addAll(priority.map(addFile))
+      ..forEach((file) {
+        _updateFile(file, (f) => f._prioritized = true);
+      });
+  }
+
+  void notifyFileChanged(String path) {
+    final tracked = addFile(path);
+    tracked._currentResult = null;
+    _putInQueue(tracked);
+  }
+
+  Future<MoorAnalysisResults> results(String path) async {
+    final tracked = addFile(path);
+
+    if (tracked._currentResult != null) {
+      return tracked._currentResult;
+    } else {
+      final completer = Completer<MoorAnalysisResults>();
+      tracked._waiting.add(completer);
+      return completer.future;
+    }
+  }
+
+  void work(Future<MoorAnalysisResults> Function(String path) worker) {
+    if (_pendingWork.isNotEmpty) {
+      final unit = _pendingWork.removeFirst();
+
+      worker(unit.path).then((result) {
+        for (var completer in unit._waiting) {
+          completer.complete(result);
+        }
+        unit._waiting.clear();
+      }, onError: (e, StackTrace s) {
+        for (var completer in unit._waiting) {
+          completer.completeError(e, s);
+        }
+        unit._waiting.clear();
+      });
+    }
+  }
+}
+
+enum FileType { moor, unknown }
+
+enum FilePriority { ignore, regular, interactive }
+
+const Map<FileType, FilePriority> _defaultPrio = {
+  FileType.moor: FilePriority.regular,
+  FileType.unknown: FilePriority.ignore,
+};
+
+class TrackedFile {
+  final String path;
+  final FileType type;
+
+  /// Whether this file has been given an elevated priority, for instance
+  /// because the user is currently typing in it.
+  bool _prioritized;
+  MoorAnalysisResults _currentResult;
+  final List<Completer<MoorAnalysisResults>> _waiting = [];
+
+  FilePriority get currentPriority =>
+      _prioritized ? FilePriority.interactive : defaultPriority;
+
+  TrackedFile._(this.path, this.type);
+
+  factory TrackedFile(String path) {
+    final type = path.endsWith('.moor') ? FileType.moor : FileType.unknown;
+    return TrackedFile._(path, type);
+  }
+
+  FilePriority get defaultPriority => _defaultPrio[type];
+}

moor_generator/pubspec.yaml

@@ -14,8 +14,9 @@ environment:
 dependencies:
   analyzer: '>=0.36.0 <0.38.0'
   analyzer_plugin:
+  collection: ^1.14.0
   recase: ^2.0.1
-  built_value: '>=6.3.0 <7.0.0'
+  built_value: ^6.3.0
   source_gen: ^0.9.4
   source_span: ^1.5.5
   build: ^1.1.0
@@ -28,7 +29,7 @@ dev_dependencies:
   test_api: ^0.2.0
   test_core: ^0.2.0
   build_runner: '>=1.1.0 <1.6.0'
-  built_value_generator: '>=6.3.0 <7.0.0'
+  built_value_generator: ^6.3.0
   build_test: '>=0.10.0 <0.11.0'
 
 dependency_overrides:

sqlparser/lib/sqlparser.dart

@@ -5,4 +5,4 @@ export 'src/analysis/analysis.dart';
 export 'src/ast/ast.dart';
 export 'src/engine/sql_engine.dart';
 export 'src/reader/parser/parser.dart' show ParsingError;
-export 'src/reader/tokenizer/token.dart' show CumulatedTokenizerException;
+export 'src/reader/tokenizer/token.dart' hide keywords;

sqlparser/lib/src/engine/sql_engine.dart

@@ -44,14 +44,13 @@ class SqlEngine {
     final parser = Parser(tokens);
 
     final stmt = parser.statement();
-    return ParseResult._(stmt, parser.errors, sql);
+    return ParseResult._(stmt, tokens, parser.errors, sql);
   }
 
   /// Parses multiple sql statements, separated by a semicolon. All
   /// [ParseResult] entries will have the same [ParseResult.errors], but the
   /// [ParseResult.sql] will only refer to the substring creating a statement.
-  List<ParseResult> parseMultiple(String sql) {
+  List<ParseResult> parseMultiple(List<Token> tokens, String sql) {
-    final tokens = tokenize(sql);
     final parser = Parser(tokens);
 
     final stmts = parser.statements();
@@ -61,7 +60,7 @@ class SqlEngine {
       final last = statement.lastPosition;
 
       final source = sql.substring(first, last);
-      return ParseResult._(statement, parser.errors, source);
+      return ParseResult._(statement, tokens, parser.errors, source);
     }).toList();
   }
 
@@ -113,6 +112,9 @@ class ParseResult {
   /// The topmost node in the sql AST that was parsed.
   final AstNode rootNode;
 
+  /// The tokens that were scanned in the source file
+  final List<Token> tokens;
+
   /// A list of all errors that occurred during parsing. [ParsingError.toString]
   /// returns a helpful description of what went wrong, along with the position
   /// where the error occurred.
@@ -121,5 +123,5 @@ class ParseResult {
   /// The sql source that created the AST at [rootNode].
   final String sql;
 
-  ParseResult._(this.rootNode, this.errors, this.sql);
+  ParseResult._(this.rootNode, this.tokens, this.errors, this.sql);
 }