Merge branch 'delightful-queries' into develop

README.md

@@ -10,4 +10,10 @@
 Moor is an easy to use, reactive persistence library for Flutter apps. Define your database tables in pure Dart and 
 enjoy a fluent query API, auto-updating streams and more!
 
-For more information, check out the [docs](https://moor.simonbinder.eu/).
+For more information, check out the [docs](https://moor.simonbinder.eu/).
+
+-----
+
+The `sqlparser` directory contains an sql parser and static analyzer, written in pure Dart.
+At the moment, it can only parse a small subset of sqlite, but most select statements are 
+supported. It hasn't been published yet as it's very experimental.

sqlparser/README.md

@@ -1,3 +1,22 @@
 # sqlparser
 
-Parser and analyzer for sql queries, written in pure Dart. Heavy work in progress
+An sql parser and static analyzer, written in pure Dart. Currently in development and
+not really suitable for any use.
+
+## Using this library
+
+```dart
+import 'package:sqlparser/sqlparser.dart';
+
+final engine = SqlEngine();
+final stmt = engine.parse('''
+SELECT f.* FROM frameworks f
+  INNER JOIN uses_language ul ON ul.framework = f.id
+  INNER JOIN languages l ON l.id = ul.language
+WHERE l.name = 'Dart'
+ORDER BY f.name ASC, f.popularity DESC
+LIMIT 5 OFFSET 5 * 3
+  ''');
+// ???
+profit();
+```

sqlparser/lib/sqlparser.dart

@@ -2,3 +2,6 @@
 ///
 /// More dartdocs go here.
 library sqlparser;
+
+export 'src/ast/ast.dart';
+export 'src/engine/sql_engine.dart';

sqlparser/lib/src/ast/ast.dart

@@ -4,6 +4,9 @@ import 'package:sqlparser/src/reader/tokenizer/token.dart';
 part 'clauses/limit.dart';
 part 'clauses/ordering.dart';
 
+part 'common/queryables.dart';
+part 'common/renamable.dart';
+
 part 'expressions/expressions.dart';
 part 'expressions/literals.dart';
 part 'expressions/reference.dart';
@@ -28,6 +31,8 @@ abstract class AstVisitor<T> {
   T visitOrderBy(OrderBy e);
   T visitOrderingTerm(OrderingTerm e);
   T visitLimit(Limit e);
+  T visitQueryable(Queryable e);
+  T visitJoin(Join e);
 
   T visitBinaryExpression(BinaryExpression e);
   T visitUnaryExpression(UnaryExpression e);

sqlparser/lib/src/ast/common/queryables.dart

@@ -0,0 +1,146 @@
+part of '../ast.dart';
+
+/// Marker interface for something that can appear after a "FROM" in a select
+/// statement.
+@sealed
+abstract class Queryable extends AstNode {
+  @override
+  T accept<T>(AstVisitor<T> visitor) => visitor.visitQueryable(this);
+
+  T when<T>({
+    @required T Function(TableReference) isTable,
+    @required T Function(SelectStatementAsSource) isSelect,
+    @required T Function(JoinClause) isJoin,
+  }) {
+    if (this is TableReference) {
+      return isTable(this as TableReference);
+    } else if (this is SelectStatementAsSource) {
+      return isSelect(this as SelectStatementAsSource);
+    } else if (this is JoinClause) {
+      return isJoin(this as JoinClause);
+    }
+
+    throw StateError('Unknown subclass');
+  }
+}
+
+/// https://www.sqlite.org/syntax/table-or-subquery.html
+/// Marker interface
+abstract class TableOrSubquery extends Queryable {}
+
+/// A table. The first path in https://www.sqlite.org/syntax/table-or-subquery.html
+class TableReference extends TableOrSubquery implements Renamable {
+  final String tableName;
+  @override
+  final String as;
+
+  TableReference(this.tableName, this.as);
+
+  @override
+  Iterable<AstNode> get childNodes => const [];
+
+  @override
+  bool contentEquals(TableReference other) {
+    return other.tableName == tableName && other.as == as;
+  }
+}
+
+/// A nested select statement.
+class SelectStatementAsSource extends TableOrSubquery implements Renamable {
+  @override
+  final String as;
+  final SelectStatement statement;
+
+  SelectStatementAsSource({@required this.statement, this.as});
+
+  @override
+  Iterable<AstNode> get childNodes => [statement];
+
+  @override
+  bool contentEquals(SelectStatementAsSource other) {
+    return other.as == as;
+  }
+}
+
+/// https://www.sqlite.org/syntax/join-clause.html
+class JoinClause extends Queryable {
+  final TableOrSubquery primary;
+  final List<Join> joins;
+
+  JoinClause({@required this.primary, @required this.joins});
+
+  @override
+  Iterable<AstNode> get childNodes => [primary, ...joins];
+
+  @override
+  bool contentEquals(JoinClause other) {
+    return true; // equality is defined by child nodes
+  }
+}
+
+enum JoinOperator {
+  comma,
+  left,
+  leftOuter,
+  inner,
+  cross,
+}
+
+class Join implements AstNode {
+  final bool natural;
+  final JoinOperator operator;
+  final TableOrSubquery query;
+  final JoinConstraint constraint;
+
+  Join(
+      {this.natural = false,
+      @required this.operator,
+      @required this.query,
+      @required this.constraint});
+
+  @override
+  Iterable<AstNode> get childNodes {
+    return [
+      query,
+      if (constraint is OnConstraint) (constraint as OnConstraint).expression
+    ];
+  }
+
+  @override
+  bool contentEquals(Join other) {
+    if (other.natural != natural || other.operator != operator) {
+      return false;
+    }
+
+    if (constraint is OnConstraint) {
+      return other.constraint is OnConstraint;
+    } else if (constraint is UsingConstraint) {
+      final typedConstraint = constraint as UsingConstraint;
+      if (other.constraint is! UsingConstraint) {
+        return false;
+      }
+      final typedOther = other.constraint as UsingConstraint;
+
+      return typedConstraint.columnNames == typedOther.columnNames;
+    }
+    return true;
+  }
+
+  @override
+  T accept<T>(AstVisitor<T> visitor) => visitor.visitJoin(this);
+}
+
+/// https://www.sqlite.org/syntax/join-constraint.html
+@sealed
+abstract class JoinConstraint {}
+
+class OnConstraint extends JoinConstraint {
+  final Expression expression;
+  OnConstraint({@required this.expression});
+}
+
+class UsingConstraint extends JoinConstraint {
+  final List<String> columnNames;
+
+  UsingConstraint({@required this.columnNames});
+}

sqlparser/lib/src/ast/common/renamable.dart

@@ -0,0 +1,5 @@
+part of '../ast.dart';
+
+abstract class Renamable {
+  String get as;
+}

sqlparser/lib/src/ast/statements/select.dart

@@ -3,10 +3,12 @@ part of '../ast.dart';
 class SelectStatement extends AstNode {
   final Expression where;
   final List<ResultColumn> columns;
+  final List<Queryable> from;
   final OrderBy orderBy;
   final Limit limit;
 
-  SelectStatement({this.where, this.columns, this.orderBy, this.limit});
+  SelectStatement(
+      {this.where, this.columns, this.from, this.orderBy, this.limit});
 
   @override
   T accept<T>(AstVisitor<T> visitor) {
@@ -50,8 +52,9 @@ class StarResultColumn extends ResultColumn {
   }
 }
 
-class ExpressionResultColumn extends ResultColumn {
+class ExpressionResultColumn extends ResultColumn implements Renamable {
   final Expression expression;
+  @override
   final String as;
 
   ExpressionResultColumn({@required this.expression, this.as});

sqlparser/lib/src/engine/sql_engine.dart

@@ -0,0 +1,16 @@
+import 'package:sqlparser/src/ast/ast.dart';
+import 'package:sqlparser/src/reader/parser/parser.dart';
+import 'package:sqlparser/src/reader/tokenizer/scanner.dart';
+
+class SqlEngine {
+  /// Parses the [sql] statement. At the moment, only SELECT statements are
+  /// supported.
+  AstNode parse(String sql) {
+    final scanner = Scanner(sql);
+    final tokens = scanner.scanTokens();
+    // todo error handling from scanner
+
+    final parser = Parser(tokens);
+    return parser.select();
+  }
+}

sqlparser/lib/src/reader/parser/parser.dart

@@ -17,6 +17,13 @@ const _binaryOperators = const [
   TokenType.pipe,
 ];
 
+final _startOperators = const [
+  TokenType.natural,
+  TokenType.left,
+  TokenType.inner,
+  TokenType.cross
+];
+
 class ParsingError implements Exception {
   final Token token;
   final String message;
@@ -53,6 +60,14 @@ class Parser {
     return false;
   }
 
+  bool _matchOne(TokenType type) {
+    if (_check(type)) {
+      _advance();
+      return true;
+    }
+    return false;
+  }
+
   bool _check(TokenType type) {
     if (_isAtEnd) return false;
     return _peek.type == type;
@@ -85,18 +100,24 @@ class Parser {
   SelectStatement select() {
     if (!_match(const [TokenType.select])) return null;
 
-    // todo parse result column
     final resultColumns = <ResultColumn>[];
     do {
       resultColumns.add(_resultColumn());
     } while (_match(const [TokenType.comma]));
 
+    final from = _from();
+
     final where = _where();
     final orderBy = _orderBy();
     final limit = _limit();
 
     return SelectStatement(
-        where: where, columns: resultColumns, orderBy: orderBy, limit: limit);
+      columns: resultColumns,
+      from: from,
+      where: where,
+      orderBy: orderBy,
+      limit: limit,
+    );
   }
 
   /// Parses a [ResultColumn] or throws if none is found.
@@ -125,17 +146,148 @@ class Parser {
     }
 
     final expr = expression();
-    // todo in sqlite, the as is optional
+    final as = _as();
+
+    return ExpressionResultColumn(expression: expr, as: as?.identifier);
+  }
+
+  /// Returns an identifier followed after an optional "AS" token in sql.
+  /// Returns null if there is
+  IdentifierToken _as() {
     if (_match(const [TokenType.as])) {
-      if (_match(const [TokenType.identifier])) {
-        final identifier = (_previous as IdentifierToken).identifier;
-        return ExpressionResultColumn(expression: expr, as: identifier);
+      return _consume(TokenType.identifier, 'Expected an identifier')
+          as IdentifierToken;
+    } else if (_match(const [TokenType.identifier])) {
+      return _previous as IdentifierToken;
+    } else {
+      return null;
+    }
+  }
+
+  List<Queryable> _from() {
+    if (!_matchOne(TokenType.from)) return [];
+
+    // Can either be a list of <TableOrSubquery> or a join. Joins also start
+    // with a TableOrSubquery, so let's first parse that.
+    final start = _tableOrSubquery();
+    // parse join, if it is one
+    final join = _joinClause(start);
+    if (join != null) {
+      return [join];
+    }
+
+    // not a join. Keep the TableOrSubqueries coming!
+    final queries = [start];
+    while (_matchOne(TokenType.comma)) {
+      queries.add(_tableOrSubquery());
+    }
+
+    return queries;
+  }
+
+  TableOrSubquery _tableOrSubquery() {
+    //  this is what we're parsing: https://www.sqlite.org/syntax/table-or-subquery.html
+    // we currently only support regular tables and nested selects
+    if (_matchOne(TokenType.identifier)) {
+      // ignore the schema name, it's not supported. Besides that, we're on the
+      // first branch in the diagram here
+      final tableName = (_previous as IdentifierToken).identifier;
+      final alias = _as();
+      return TableReference(tableName, alias?.identifier);
+    } else if (_matchOne(TokenType.leftParen)) {
+      final innerStmt = select();
+      _consume(TokenType.rightParen,
+          'Expected a right bracket to terminate the inner select');
+
+      final alias = _as();
+      return SelectStatementAsSource(
+          statement: innerStmt, as: alias?.identifier);
+    }
+
+    _error('Expected a table name or a nested select statement');
+  }
+
+  JoinClause _joinClause(TableOrSubquery start) {
+    var operator = _parseJoinOperatorNoComma();
+    if (operator == null) {
+      return null;
+    }
+
+    final joins = <Join>[];
+
+    while (operator != null) {
+      final subquery = _tableOrSubquery();
+      final constraint = _joinConstraint();
+      JoinOperator resolvedOperator;
+      if (operator.contains(TokenType.left)) {
+        resolvedOperator = operator.contains(TokenType.outer)
+            ? JoinOperator.leftOuter
+            : JoinOperator.left;
+      } else if (operator.contains(TokenType.inner)) {
+        resolvedOperator = JoinOperator.inner;
+      } else if (operator.contains(TokenType.cross)) {
+        resolvedOperator = JoinOperator.cross;
+      } else if (operator.contains(TokenType.comma)) {
+        resolvedOperator = JoinOperator.comma;
+      }
+
+      joins.add(Join(
+        natural: operator.contains(TokenType.natural),
+        operator: resolvedOperator,
+        query: subquery,
+        constraint: constraint,
+      ));
+
+      // parse the next operator, if there is more than one join
+      if (_matchOne(TokenType.comma)) {
+        operator = [TokenType.comma];
       } else {
-        throw ParsingError(_peek, 'Expected an identifier as the column name');
+        operator = _parseJoinOperatorNoComma();
       }
     }
 
-    return ExpressionResultColumn(expression: expr);
+    return JoinClause(primary: start, joins: joins);
+  }
+
+  /// Parses https://www.sqlite.org/syntax/join-operator.html, minus the comma.
+  List<TokenType> _parseJoinOperatorNoComma() {
+    if (_match(_startOperators)) {
+      final operators = [_previous.type];
+      // natural is a prefix, another operator can follow.
+      if (_previous.type == TokenType.natural) {
+        if (_match([TokenType.left, TokenType.inner, TokenType.cross])) {
+          operators.add(_previous.type);
+        }
+      }
+      if (_previous.type == TokenType.left && _matchOne(TokenType.outer)) {
+        operators.add(_previous.type);
+      }
+
+      _consume(TokenType.join, 'Expected to see a join keyword here');
+      return operators;
+    }
+    return null;
+  }
+
+  /// Parses https://www.sqlite.org/syntax/join-constraint.html
+  JoinConstraint _joinConstraint() {
+    if (_matchOne(TokenType.on)) {
+      return OnConstraint(expression: expression());
+    } else if (_matchOne(TokenType.using)) {
+      _consume(TokenType.leftParen, 'Expected an opening paranthesis');
+
+      final columnNames = <String>[];
+      do {
+        final identifier =
+            _consume(TokenType.identifier, 'Expected a column name');
+        columnNames.add((identifier as IdentifierToken).identifier);
+      } while (_matchOne(TokenType.comma));
+
+      _consume(TokenType.rightParen, 'Expected an closing paranthesis');
+
+      return UsingConstraint(columnNames: columnNames);
+    }
+    _error('Expected a constraint with ON or USING');
   }
 
   /// Parses a where clause if there is one at the current position
@@ -152,7 +304,8 @@ class Parser {
       final terms = <OrderingTerm>[];
       do {
         terms.add(_orderingTerm());
-      } while (_match(const [TokenType.comma]));
+      } while (_matchOne(TokenType.comma));
+      return OrderBy(terms: terms);
     }
     return null;
   }
@@ -301,19 +454,18 @@ class Parser {
     final type = token.type;
     switch (type) {
       case TokenType.numberLiteral:
-        return NumericLiteral(_parseNumber(token.lexeme), _peek);
+        return NumericLiteral(_parseNumber(token.lexeme), token);
       case TokenType.stringLiteral:
-        final token = _peek as StringLiteralToken;
-        return StringLiteral(token);
+        return StringLiteral(token as StringLiteralToken);
       case TokenType.$null:
-        return NullLiteral(_peek);
+        return NullLiteral(token);
       case TokenType.$true:
-        return BooleanLiteral.withTrue(_peek);
+        return BooleanLiteral.withTrue(token);
       case TokenType.$false:
-        return BooleanLiteral.withFalse(_peek);
+        return BooleanLiteral.withFalse(token);
       // todo CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP
       case TokenType.leftParen:
-        final left = _previous;
+        final left = token;
         final expr = expression();
         _consume(TokenType.rightParen, 'Expected a closing bracket');
         return Parentheses(left, expr, _previous);

sqlparser/lib/src/reader/tokenizer/token.dart

@@ -45,11 +45,19 @@ enum TokenType {
   identifier,
 
   select,
-
   from,
   as,
   where,
 
+  natural,
+  left,
+  outer,
+  inner,
+  cross,
+  join,
+  on,
+  using,
+
   order,
   by,
   asc,
@@ -64,6 +72,14 @@ enum TokenType {
 const Map<String, TokenType> keywords = {
   'SELECT': TokenType.select,
   'FROM': TokenType.from,
+  'NATURAL': TokenType.natural,
+  'LEFT': TokenType.leftParen,
+  'OUTER': TokenType.outer,
+  'INNER': TokenType.inner,
+  'CROSS': TokenType.cross,
+  'JOIN': TokenType.join,
+  'ON': TokenType.on,
+  'USING': TokenType.using,
   'AS': TokenType.as,
   'WHERE': TokenType.where,
   'ORDER': TokenType.order,

sqlparser/pubspec.yaml

@@ -8,6 +8,7 @@ author: Simon Binder <oss@simonbinder.eu>
 
 environment:
   sdk: '>=2.2.2 <3.0.0'
+  meta: ^1.1.7
 
 dev_dependencies:
   test: ^1.0.0

sqlparser/test/parser/expression_test.dart

@@ -62,4 +62,19 @@ void main() {
       ),
     );
   });
+
+  test('pars', () {
+    final scanner = Scanner('''
+SELECT f.* FROM frameworks f
+  INNER JOIN uses_language ul ON ul.framework = f.id
+  INNER JOIN languages l ON l.id = ul.language
+WHERE l.name = 'Dart'
+ORDER BY f.name ASC, f.popularity DESC
+LIMIT 5 OFFSET 5 * 3
+    ''');
+    final tokens = scanner.scanTokens();
+    final parser = Parser(tokens);
+    final stmt = parser.select();
+    print(stmt);
+  });
 }