Analysis for sqlite 3.38

.github/workflows/main.yml

@@ -12,7 +12,7 @@ jobs:
     name: "Compile sqlite3 for tests"
     runs-on: ubuntu-20.04
     env:
-      SQLITE_VERSION: "3370000"
+      SQLITE_VERSION: "3380000"
 
     steps:
       - uses: actions/checkout@v2

drift_dev/lib/src/analyzer/errors.dart

@@ -93,13 +93,16 @@ class ErrorInMoorFile extends MoorError {
     var msg = error.message ?? error.type.toString();
     if (error.type == AnalysisErrorType.notSupportedInDesiredVersion) {
       msg = '$msg\nNote: You can change the sqlite version with build options. '
-          'See https://moor.simonbinder.eu/options/ for details!';
+          'See https://drift.simonbinder.eu/options/ for details!';
     }
 
+    final defaultSeverity =
+        error.type == AnalysisErrorType.hint ? Severity.hint : Severity.error;
+
     return ErrorInMoorFile(
       span: error.span!,
       message: msg,
-      severity: overrideSeverity ?? Severity.error,
+      severity: overrideSeverity ?? defaultSeverity,
     );
   }
 

sqlparser/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.21.0
+
+- Analysis support for new features in sqlite version 3.38.
+
 ## 0.20.1
 
 - Fix SQL generation for upsert statements with a conflict target.

sqlparser/lib/src/analysis/error.dart

@@ -81,4 +81,5 @@ enum AnalysisErrorType {
   noTypeNameInStrictTable,
   invalidTypeNameInStrictTable,
   other,
+  hint,
 }

sqlparser/lib/src/analysis/steps/linting_visitor.dart

@@ -10,6 +10,22 @@ class LintingVisitor extends RecursiveVisitor<void, void> {
 
   LintingVisitor(this.options, this.context);
 
+  @override
+  void visitBinaryExpression(BinaryExpression e, void arg) {
+    final operator = e.operator.type;
+    if ((operator == TokenType.dashRangle ||
+            operator == TokenType.dashRangleRangle) &&
+        options.version < SqliteVersion.v3_38) {
+      context.reportError(AnalysisError(
+        type: AnalysisErrorType.notSupportedInDesiredVersion,
+        message: '`->` and `->>` require sqlite3 version 38',
+        relevantNode: e.operator,
+      ));
+    }
+
+    visitChildren(e, arg);
+  }
+
   @override
   void visitCommonTableExpression(CommonTableExpression e, void arg) {
     if (e.materializationHint != null &&
@@ -195,6 +211,35 @@ class LintingVisitor extends RecursiveVisitor<void, void> {
       options.addedFunctions[lowercaseCall]!.reportErrors(e, context);
     }
 
+    switch (e.name.toLowerCase()) {
+      case 'format':
+      case 'unixepoch':
+        // These were added in sqlite3 version 3.38
+        if (options.version < SqliteVersion.v3_38) {
+          context.reportError(
+            AnalysisError(
+              type: AnalysisErrorType.notSupportedInDesiredVersion,
+              message: 'The `${e.name}` function is not available in '
+                  '${options.version}.',
+              relevantNode: e,
+            ),
+          );
+        }
+        break;
+      case 'printf':
+        // `printf` was renamed to `format` in sqlite3 version 3.38
+        if (options.version >= SqliteVersion.v3_38) {
+          context.reportError(
+            AnalysisError(
+              type: AnalysisErrorType.hint,
+              message: '`printf` was renamed to `format()`, consider using '
+                  'that function instead.',
+              relevantNode: e,
+            ),
+          );
+        }
+    }
+
     visitChildren(e, arg);
   }
 

sqlparser/lib/src/analysis/types/resolving_visitor.dart

@@ -323,13 +323,26 @@ class TypeResolver extends RecursiveVisitor<TypeExpectation, void> {
         break;
       case TokenType.doublePipe:
         // string concatenation.
-        const stringType = ResolvedType(type: BasicType.text);
-        session._checkAndResolve(e, stringType, arg);
+        session._checkAndResolve(e, _textType, arg);
         session._addRelation(NullableIfSomeOtherIs(e, [e.left, e.right]));
-        const childExpectation = ExactTypeExpectation.laxly(stringType);
+        const childExpectation = ExactTypeExpectation.laxly(_textType);
         visit(e.left, childExpectation);
         visit(e.right, childExpectation);
         break;
+      case TokenType.dashRangle:
+        // Extract as JSON, this takes two strings and returns a string (or
+        // `NULL` if the value wasn't found).
+        session._checkAndResolve(e, _textType.withNullable(true), arg);
+        visit(e.left, _expectString);
+        visit(e.right, _expectString);
+        break;
+      case TokenType.dashRangleRangle:
+        // Extract as JSON to SQL value.
+        session._hintNullability(e, true);
+
+        visit(e.left, _expectString);
+        visit(e.right, _expectString);
+        break;
       default:
         throw StateError('Binary operator ${e.operator.type} not recognized '
             'by types2. At $e');
@@ -482,6 +495,7 @@ class TypeResolver extends RecursiveVisitor<TypeExpectation, void> {
       case 'lower':
       case 'ltrim':
       case 'printf':
+      case 'format':
       case 'replace':
       case 'rtrim':
       case 'substr':
@@ -496,6 +510,7 @@ class TypeResolver extends RecursiveVisitor<TypeExpectation, void> {
       case 'datetime':
       case 'julianday':
       case 'strftime':
+      case 'unixepoch':
       case 'char':
       case 'hex':
       case 'quote':

sqlparser/lib/src/engine/options.dart

@@ -11,7 +11,7 @@ class EngineOptions {
   ///
   /// The library will report when using sqlite features that were added after
   /// the desired [version].
-  /// Defaults to [SqliteVersion.current].
+  /// Defaults to [SqliteVersion.minimum].
   final SqliteVersion version;
 
   /// All [Extension]s that have been enabled in this sql engine.
@@ -28,9 +28,9 @@ class EngineOptions {
 
   EngineOptions({
     this.useMoorExtensions = false,
-    this.enabledExtensions = const [],
+    List<Extension> enabledExtensions = const [],
     this.version = SqliteVersion.minimum,
-  }) {
+  }) : enabledExtensions = _allExtensions(enabledExtensions, version) {
     if (version < SqliteVersion.minimum) {
       throw ArgumentError.value(
           version, 'version', 'Must at least be ${SqliteVersion.minimum}');
@@ -41,6 +41,18 @@ class EngineOptions {
     }
   }
 
+  static List<Extension> _allExtensions(
+      List<Extension> added, SqliteVersion version) {
+    return [
+      // The json1 extension was enabled by default in sqlite3 version 3.38, so
+      // add it if it's not already enabled.
+      if (version >= SqliteVersion.v3_38 &&
+          !added.any((e) => e is Json1Extension))
+        const Json1Extension(),
+      ...added,
+    ];
+  }
+
   void addFunctionHandler(FunctionHandler handler) {
     _addedFunctionHandlers.add(handler);
 
@@ -67,6 +79,9 @@ class SqliteVersion implements Comparable<SqliteVersion> {
   /// can't provide analysis warnings when using recent sqlite3 features.
   static const SqliteVersion minimum = SqliteVersion.v3(34);
 
+  /// Version `3.38.0` of `sqlite3`.
+  static const SqliteVersion v3_38 = SqliteVersion.v3(38);
+
   /// Version `3.37.0` of `sqlite3`.
   static const SqliteVersion v3_37 = SqliteVersion.v3(37);
 
@@ -76,7 +91,7 @@ class SqliteVersion implements Comparable<SqliteVersion> {
   /// The highest sqlite version supported by this `sqlparser` package.
   ///
   /// Newer features in `sqlite3` may not be recognized by this library.
-  static const SqliteVersion current = v3_37;
+  static const SqliteVersion current = v3_38;
 
   /// The major version of sqlite.
   ///

sqlparser/lib/src/reader/parser.dart

@@ -582,7 +582,11 @@ class Parser {
   }
 
   Expression _concatenation() {
-    return _parseSimpleBinary(const [TokenType.doublePipe], _unary);
+    return _parseSimpleBinary(const [
+      TokenType.doublePipe,
+      TokenType.dashRangle,
+      TokenType.dashRangleRangle
+    ], _unary);
   }
 
   Expression _unary() {

sqlparser/lib/src/reader/tokenizer/scanner.dart

@@ -78,6 +78,10 @@ class Scanner {
       case $minus:
         if (_match($minus)) {
           _lineComment();
+        } else if (_match($rangle)) {
+          _addToken(_match($rangle)
+              ? TokenType.dashRangleRangle
+              : TokenType.dashRangle);
         } else {
           _addToken(TokenType.minus);
         }

sqlparser/lib/src/reader/tokenizer/token.dart

@@ -63,6 +63,12 @@ enum TokenType {
   currentTimestamp,
   currentUser,
   database,
+
+  /// `->`, extract subcomponent of JSON
+  dashRangle,
+
+  /// `->>`, extract subcomponent of JSON as SQL value.
+  dashRangleRangle,
   deferrable,
   deferred,
   delete,

sqlparser/lib/utils/node_to_text.dart

@@ -105,6 +105,8 @@ class NodeSqlBuilder extends AstVisitor<void, void> {
       TokenType.doubleEqual: '==',
       TokenType.exclamationEqual: '!=',
       TokenType.lessMore: '<>',
+      TokenType.dashRangle: '->',
+      TokenType.dashRangleRangle: '->>',
     }[e.operator.type];
 
     if (operatorSymbol != null) {

sqlparser/pubspec.yaml

@@ -1,6 +1,6 @@
 name: sqlparser
 description: Parses sqlite statements and performs static analysis on them
-version: 0.20.1
+version: 0.21.0-dev
 homepage: https://github.com/simolus3/moor/tree/develop/sqlparser
 #homepage: https://moor.simonbinder.eu/
 issue_tracker: https://github.com/simolus3/moor/issues

sqlparser/test/analysis/errors/unsupported_test.dart

@@ -70,4 +70,39 @@ void main() {
 
     expect(currentEngine.analyze(sql).errors, isEmpty);
   });
+
+  test('does not support `->` and `->>`in old sqlite3 versions', () {
+    minimumEngine.analyze("SELECT '' -> ''").expectError('->',
+        type: AnalysisErrorType.notSupportedInDesiredVersion);
+    minimumEngine.analyze("SELECT '' ->> ''").expectError('->>',
+        type: AnalysisErrorType.notSupportedInDesiredVersion);
+
+    currentEngine.analyze("SELECT '' -> ''").expectNoError();
+    currentEngine.analyze("SELECT '' ->> ''").expectNoError();
+  });
+
+  test('warns about using `printf` after 3.38', () {
+    const sql = "SELECT printf('', 0, 'foo')";
+
+    currentEngine
+        .analyze(sql)
+        .expectError("printf('', 0, 'foo')", type: AnalysisErrorType.hint);
+    minimumEngine.analyze(sql).expectNoError();
+  });
+
+  test('warns about using unixepoch before 3.38', () {
+    const sql = "SELECT unixepoch('')";
+
+    minimumEngine.analyze(sql).expectError("unixepoch('')",
+        type: AnalysisErrorType.notSupportedInDesiredVersion);
+    currentEngine.analyze(sql).expectNoError();
+  });
+
+  test('warns about using format before 3.38', () {
+    const sql = "SELECT format('', 0, 'foo')";
+
+    minimumEngine.analyze(sql).expectError("format('', 0, 'foo')",
+        type: AnalysisErrorType.notSupportedInDesiredVersion);
+    currentEngine.analyze(sql).expectNoError();
+  });
 }

sqlparser/test/analysis/types2/misc_cases_test.dart

@@ -44,6 +44,12 @@ const Map<String, ResolvedType?> _types = {
       ResolvedType.bool(),
   'SELECT GROUP_CONCAT(content) = ? FROM demo;':
       ResolvedType(type: BasicType.text, nullable: true),
+  "SELECT '' -> '' = ?": ResolvedType(type: BasicType.text, nullable: true),
+  "SELECT '' ->> '' = ?": null,
+  "SELECT ? -> 'a' = 'b'": ResolvedType(type: BasicType.text, nullable: false),
+  "SELECT ? ->> 'a' = 'b'": ResolvedType(type: BasicType.text, nullable: false),
+  "SELECT 'a' -> ? = 'b'": ResolvedType(type: BasicType.text, nullable: false),
+  "SELECT 'a' ->> ? = 'b'": ResolvedType(type: BasicType.text, nullable: false),
 };
 
 SqlEngine _spawnEngine() {

sqlparser/test/scanner/recognizes_all_tokens_test.dart

@@ -29,7 +29,7 @@ void main() {
   }
 
   test(
-    'recognizes all sqlite tokens',
+    'recognizes all sqlite keywords',
     () {
       final keywordCount =
           library.lookupFunction<SqliteKeywordCountNative, SqliteKeywordCount>(

sqlparser/test/scanner/scanner_test.dart

@@ -31,4 +31,14 @@ void main() {
           .having((e) => e.identifier, 'identifier', 'SELECT')),
     );
   });
+
+  test('scans new tokens for JSON extraction', () {
+    expect(Scanner('- -> ->>').scanTokens(), [
+      isA<Token>().having((e) => e.type, 'tokenType', TokenType.minus),
+      isA<Token>().having((e) => e.type, 'tokenType', TokenType.dashRangle),
+      isA<Token>()
+          .having((e) => e.type, 'tokenType', TokenType.dashRangleRangle),
+      isA<Token>().having((e) => e.type, 'tokenType', TokenType.eof),
+    ]);
+  });
 }

sqlparser/test/utils/node_to_text_test.dart

@@ -431,6 +431,11 @@ CREATE UNIQUE INDEX my_idx ON t1 (c1, c2, c3) WHERE c1 < c3;
       testFormat('SELECT foo.bar');
       testFormat('SELECT foo.bar.baz');
     });
+
+    test('json', () {
+      testFormat('SELECT a -> b');
+      testFormat('SELECT a ->> b');
+    });
   });
 
   test('identifiers', () {