Parse simple expressions

sqlparser/lib/src/ast/ast.dart

@@ -0,0 +1,14 @@
+import 'package:sqlparser/src/ast/expressions/literals.dart';
+import 'package:sqlparser/src/ast/expressions/simple.dart';
+
+abstract class AstNode {
+  Iterable<AstNode> get childNodes;
+  T accept<T>(AstVisitor<T> visitor);
+}
+
+abstract class AstVisitor<T> {
+  T visitBinaryExpression(BinaryExpression e);
+  T visitUnaryExpression(UnaryExpression e);
+  T visitIsExpression(IsExpression e);
+  T visitLiteral(Literal e);
+}

sqlparser/lib/src/ast/expressions/expressions.dart

@@ -1,3 +1,5 @@
-abstract class Expression {
+import 'package:sqlparser/src/ast/ast.dart';
+
+abstract class Expression implements AstNode {
   const Expression();
 }

sqlparser/lib/src/ast/expressions/literals.dart

@@ -1,16 +1,43 @@
+import 'package:sqlparser/src/ast/ast.dart';
+import 'package:sqlparser/src/reader/tokenizer/token.dart';
+
 import 'expressions.dart';
 
 // https://www.sqlite.org/syntax/literal-value.html
 
-class NullLiteral extends Expression {
-  const NullLiteral();
+abstract class Literal extends Expression {
+  final Token token;
+
+  Literal(this.token);
 
   @override
-  String toString() => 'NULL';
+  T accept<T>(AstVisitor<T> visitor) => visitor.visitLiteral(this);
 
   @override
-  int get hashCode => runtimeType.hashCode;
-
-  @override
-  bool operator ==(other) => identical(this, other) || other is NullLiteral;
+  final Iterable<AstNode> childNodes = const <AstNode>[];
+}
+
+class NullLiteral extends Literal {
+  NullLiteral(Token token) : super(token);
+}
+
+class NumericLiteral extends Literal {
+  final num number;
+
+  NumericLiteral(this.number, Token token) : super(token);
+}
+
+class BooleanLiteral extends NumericLiteral {
+  BooleanLiteral.withFalse(Token token) : super(0, token);
+  BooleanLiteral.withTrue(Token token) : super(1, token);
+}
+
+class StringLiteral extends Literal {
+  final String data;
+  final bool isBinary;
+
+  StringLiteral(StringLiteralToken token)
+      : data = token.value,
+        isBinary = token.binary,
+        super(token);
 }

sqlparser/lib/src/ast/expressions/simple.dart

@@ -0,0 +1,63 @@
+import 'package:sqlparser/src/ast/ast.dart';
+import 'package:sqlparser/src/reader/tokenizer/token.dart';
+
+import 'expressions.dart';
+
+class UnaryExpression extends Expression {
+  final Token operator;
+  final Expression inner;
+
+  UnaryExpression(this.operator, this.inner);
+
+  @override
+  T accept<T>(AstVisitor<T> visitor) => visitor.visitUnaryExpression(this);
+
+  @override
+  Iterable<AstNode> get childNodes => [inner];
+}
+
+class BinaryExpression extends Expression {
+  final Token operator;
+  final Expression left;
+  final Expression right;
+
+  BinaryExpression(this.left, this.operator, this.right);
+
+  @override
+  T accept<T>(AstVisitor<T> visitor) => visitor.visitBinaryExpression(this);
+
+  @override
+  Iterable<AstNode> get childNodes => [left, right];
+}
+
+class IsExpression extends Expression {
+  final bool negated;
+  final Expression left;
+  final Expression right;
+
+  IsExpression(this.negated, this.left, this.right);
+
+  @override
+  T accept<T>(AstVisitor<T> visitor) {
+    return visitor.visitIsExpression(this);
+  }
+
+  @override
+  Iterable<AstNode> get childNodes => [left, right];
+}
+
+class Parentheses extends Expression {
+  final Token openingLeft;
+  final Expression expression;
+  final Token closingRight;
+
+  Parentheses(this.openingLeft, this.expression, this.closingRight);
+
+  @override
+  T accept<T>(AstVisitor<T> visitor) {
+    return expression.accept(visitor);
+  }
+
+  @override
+  Iterable<AstNode> get childNodes => [expression];
+}

sqlparser/lib/src/ast/expressions/unary.dart

@@ -1,10 +0,0 @@
-import 'package:sqlparser/src/reader/tokenizer/token.dart';
-
-import 'expressions.dart';
-
-class UnaryExpression extends Expression {
-  final Token operator;
-  final Expression inner;
-
-  UnaryExpression(this.operator, this.inner);
-}

sqlparser/lib/src/reader/parser/parser.dart

@@ -1,9 +1,35 @@
+import 'package:meta/meta.dart';
 import 'package:sqlparser/src/ast/expressions/expressions.dart';
-import 'package:sqlparser/src/ast/expressions/unary.dart';
+import 'package:sqlparser/src/ast/expressions/literals.dart';
+import 'package:sqlparser/src/ast/expressions/simple.dart';
 import 'package:sqlparser/src/reader/tokenizer/token.dart';
 
+const _comparisonOperators = [
+  TokenType.less,
+  TokenType.lessEqual,
+  TokenType.more,
+  TokenType.moreEqual,
+];
+const _binaryOperators = const [
+  TokenType.shiftLeft,
+  TokenType.shiftRight,
+  TokenType.ampersand,
+  TokenType.pipe,
+];
+
+class ParsingError implements Exception {
+  final Token token;
+  final String message;
+
+  ParsingError(this.token, this.message);
+}
+
+// todo better error handling and synchronisation, like it's done here:
+// https://craftinginterpreters.com/parsing-expressions.html#synchronizing-a-recursive-descent-parser
+
 class Parser {
   final List<Token> tokens;
+  final List<ParsingError> errors = [];
   int _current = 0;
 
   Parser(this.tokens);
@@ -34,6 +60,18 @@ class Parser {
     return _previous;
   }
 
+  @alwaysThrows
+  void _error(String message) {
+    final error = ParsingError(_peek, message);
+    errors.add(error);
+    throw error;
+  }
+
+  Token _consume(TokenType type, String message) {
+    if (_check(type)) return _advance();
+    _error(message);
+  }
+
   /* We parse expressions here.
   * Operators have the following precedence:
   *  - + ~ NOT (unary)
@@ -48,10 +86,84 @@ class Parser {
   *  We also treat expressions in parentheses and literals with the highest
   *  priority. Parsing methods are written in ascending precedence, and each
   *  parsing method calls the next higher precedence if unsuccessful.
+  *  https://www.sqlite.org/lang_expr.html
   * */
 
   Expression expression() {
-    return _unary();
+    return _or();
+  }
+
+  /// Parses an expression of the form a <T> b, where <T> is in [types] and
+  /// both a and b are expressions with a higher precedence parsed from
+  /// [higherPrecedence].
+  Expression _parseSimpleBinary(
+      List<TokenType> types, Expression Function() higherPrecedence) {
+    var expression = higherPrecedence();
+
+    while (_match(types)) {
+      final operator = _previous;
+      final right = higherPrecedence();
+      expression = BinaryExpression(expression, operator, right);
+    }
+    return expression;
+  }
+
+  Expression _or() => _parseSimpleBinary(const [TokenType.or], _and);
+  Expression _and() => _parseSimpleBinary(const [TokenType.and], _equals);
+
+  Expression _equals() {
+    var expression = _comparison();
+    final ops = const [
+      TokenType.equal,
+      TokenType.doubleEqual,
+      TokenType.exclamationEqual,
+      TokenType.lessMore,
+      TokenType.$is,
+      TokenType.$in,
+      TokenType.like,
+      TokenType.glob,
+      TokenType.match,
+      TokenType.regexp,
+    ];
+
+    while (_match(ops)) {
+      final operator = _previous;
+      if (operator.type == TokenType.$is) {
+        final not = _match(const [TokenType.not]);
+        // special case: is not expression
+        expression = IsExpression(not, expression, _comparison());
+      } else {
+        expression = BinaryExpression(expression, operator, _comparison());
+      }
+    }
+    return expression;
+  }
+
+  Expression _comparison() {
+    return _parseSimpleBinary(_comparisonOperators, _binaryOperation);
+  }
+
+  Expression _binaryOperation() {
+    return _parseSimpleBinary(_binaryOperators, _addition);
+  }
+
+  Expression _addition() {
+    return _parseSimpleBinary(const [
+      TokenType.plus,
+      TokenType.minus,
+    ], _multiplication);
+  }
+
+  Expression _multiplication() {
+    return _parseSimpleBinary(const [
+      TokenType.star,
+      TokenType.slash,
+      TokenType.percent,
+    ], _concatenation);
+  }
+
+  Expression _concatenation() {
+    return _parseSimpleBinary(const [TokenType.doublePipe], _unary);
   }
 
   Expression _unary() {
@@ -70,6 +182,32 @@ class Parser {
   }
 
   Expression _primary() {
-    return null;
+    final token = _advance();
+    final type = token.type;
+    switch (type) {
+      case TokenType.numberLiteral:
+        // todo get the proper value out of this one
+        return NumericLiteral(42, _peek);
+      case TokenType.stringLiteral:
+        final token = _peek as StringLiteralToken;
+        return StringLiteral(token);
+      case TokenType.$null:
+        return NullLiteral(_peek);
+      case TokenType.$true:
+        return BooleanLiteral.withTrue(_peek);
+      case TokenType.$false:
+        return BooleanLiteral.withFalse(_peek);
+      // todo CURRENT_TIME, CURRENT_DATE, CURRENT_TIMESTAMP
+      case TokenType.leftParen:
+        final left = _previous;
+        final expr = expression();
+        _consume(TokenType.rightParen, 'Expected a closing bracket');
+        return Parentheses(left, expr, _previous);
+      default:
+        break;
+    }
+
+    // nothing found -> issue error
+    _error('Could not parse this expression');
   }
 }

sqlparser/lib/src/reader/tokenizer/scanner.dart

@@ -175,7 +175,7 @@ class Scanner {
     _nextChar();
 
     final value = source.substring(_startOffset + 1, _currentOffset - 1);
-    tokens.add(StringLiteral(value, _currentSpan, binary: binary));
+    tokens.add(StringLiteralToken(value, _currentSpan, binary: binary));
   }
 
   void _numeric(String firstChar) {

sqlparser/lib/src/reader/tokenizer/token.dart

@@ -36,6 +36,12 @@ enum TokenType {
 
   stringLiteral,
   numberLiteral,
+  $true,
+  $false,
+  $null,
+  currentTime,
+  currentDate,
+  currentTimestamp,
   identifier,
 
   select,
@@ -56,6 +62,12 @@ const Map<String, TokenType> keywords = {
   'MATCH': TokenType.match,
   'REGEXP': TokenType.regexp,
   'NOT': TokenType.not,
+  'TRUE': TokenType.$true,
+  'FALSE': TokenType.$false,
+  'NULL': TokenType.$null,
+  'CURRENT_TIME': TokenType.currentTime,
+  'CURRENT_DATE': TokenType.currentDate,
+  'CURRENT_TIMESTAMP': TokenType.currentTimestamp,
 };
 
 class Token {
@@ -66,13 +78,13 @@ class Token {
   const Token(this.type, this.span);
 }
 
-class StringLiteral extends Token {
+class StringLiteralToken extends Token {
   final String value;
 
   /// sqlite allows binary strings (x'literal') which are interpreted as blobs.
   final bool binary;
 
-  const StringLiteral(this.value, SourceSpan span, {this.binary = false})
+  const StringLiteralToken(this.value, SourceSpan span, {this.binary = false})
       : super(TokenType.stringLiteral, span);
 }
 

sqlparser/lib/src/utils/ast_equality.dart

@@ -0,0 +1,46 @@
+import 'package:sqlparser/src/ast/ast.dart';
+import 'package:sqlparser/src/ast/expressions/simple.dart';
+
+/// Checks whether [a] and [b] are equal. If they aren't, throws an exception.
+void enforceEqual(AstNode a, AstNode b) {
+  if (a.runtimeType != b.runtimeType) {
+    throw ArgumentError('Not equal: First was $a, second $b');
+  }
+  _checkAdditional(a, b);
+
+  final childrenA = a.childNodes.iterator;
+  final childrenB = b.childNodes.iterator;
+
+  // always move both iterators
+  while (childrenA.moveNext() & childrenB.moveNext()) {
+    enforceEqual(childrenA.current, childrenB.current);
+  }
+
+  if (childrenA.moveNext() || childrenB.moveNext()) {
+    throw ArgumentError("$a and $b don't have an equal amount of children");
+  }
+}
+
+void _checkAdditional<T extends AstNode>(T a, T b) {
+  if (a is IsExpression) {
+    return _checkIsExpr(a, b as IsExpression);
+  } else if (a is UnaryExpression) {
+    _checkUnaryExpression(a, b as UnaryExpression);
+  } else if (a is BinaryExpression) {
+    _checkBinaryExpression(a, b as BinaryExpression);
+  }
+}
+
+void _checkIsExpr(IsExpression a, IsExpression b) {
+  if (a.negated != b.negated) {
+    throw ArgumentError('Negation status not the same');
+  }
+}
+
+void _checkUnaryExpression(UnaryExpression a, UnaryExpression b) {
+  if (a.operator.type != b.operator.type) throw ArgumentError('Different type');
+}
+
+void _checkBinaryExpression(BinaryExpression a, BinaryExpression b) {
+  if (a.operator.type != b.operator.type) throw ArgumentError('Different type');
+}

sqlparser/test/parser/expression_test.dart

@@ -0,0 +1,35 @@
+import 'package:sqlparser/src/ast/expressions/literals.dart';
+import 'package:sqlparser/src/ast/expressions/simple.dart';
+import 'package:sqlparser/src/reader/parser/parser.dart';
+import 'package:sqlparser/src/reader/tokenizer/scanner.dart';
+import 'package:sqlparser/src/reader/tokenizer/token.dart';
+import 'package:sqlparser/src/utils/ast_equality.dart';
+import 'package:test/test.dart';
+
+import 'utils.dart';
+
+void main() {
+  test('parses simple expressions', () {
+    final scanner = Scanner('3 * 4 + 5 == 17');
+    final tokens = scanner.scanTokens();
+    final parser = Parser(tokens);
+
+    final expression = parser.expression();
+    enforceEqual(
+      expression,
+      BinaryExpression(
+        BinaryExpression(
+          BinaryExpression(
+            NumericLiteral(3, token(TokenType.numberLiteral)),
+            token(TokenType.star),
+            NumericLiteral(4, token(TokenType.numberLiteral)),
+          ),
+          token(TokenType.plus),
+          NumericLiteral(5, token(TokenType.numberLiteral)),
+        ),
+        token(TokenType.doubleEqual),
+        NumericLiteral(17, token(TokenType.numberLiteral)),
+      ),
+    );
+  });
+}

sqlparser/test/parser/utils.dart

@@ -0,0 +1,5 @@
+import 'package:sqlparser/src/reader/tokenizer/token.dart';
+
+Token token(TokenType type) {
+  return Token(type, null);
+}