mirror of https://github.com/AMT-Cheif/drift.git
Scan comments in sql
This commit is contained in:
parent
2bf42a6157
commit
746b8401ec
|
@ -38,11 +38,12 @@ void _visit(SpecifiedTable table, _SortRun run) {
|
||||||
}
|
}
|
||||||
|
|
||||||
class _SortRun {
|
class _SortRun {
|
||||||
Map<SpecifiedTable, SpecifiedTable> previous = {};
|
final Map<SpecifiedTable, SpecifiedTable> previous = {};
|
||||||
final List<SpecifiedTable> result = [];
|
final List<SpecifiedTable> result = [];
|
||||||
|
|
||||||
/// Throws a [CircularReferenceException] because the [last] table depends on
|
/// Throws a [CircularReferenceException] because the [last] table depends on
|
||||||
/// [first], which transitively depends on [last] first.
|
/// [first], which (transitively) depends on [last] as well. The path in the
|
||||||
|
/// thrown exception will go from [first] to [last].
|
||||||
void throwCircularException(SpecifiedTable last, SpecifiedTable first) {
|
void throwCircularException(SpecifiedTable last, SpecifiedTable first) {
|
||||||
final constructedPath = <SpecifiedTable>[];
|
final constructedPath = <SpecifiedTable>[];
|
||||||
for (var current = last; current != first; current = previous[current]) {
|
for (var current = last; current != first; current = previous[current]) {
|
||||||
|
|
|
@ -35,6 +35,11 @@ class SqlEngine {
|
||||||
|
|
||||||
/// Tokenizes the [source] into a list list [Token]s. Each [Token] contains
|
/// Tokenizes the [source] into a list list [Token]s. Each [Token] contains
|
||||||
/// information about where it appears in the [source] and a [TokenType].
|
/// information about where it appears in the [source] and a [TokenType].
|
||||||
|
///
|
||||||
|
/// Note that the list might be tokens that should be
|
||||||
|
/// [Token.invisibleToParser], if you're passing them to a [Parser] directly,
|
||||||
|
/// you need to filter them. When using the methods in this class, this will
|
||||||
|
/// be taken care of automatically.
|
||||||
List<Token> tokenize(String source) {
|
List<Token> tokenize(String source) {
|
||||||
final scanner = Scanner(source, scanMoorTokens: useMoorExtensions);
|
final scanner = Scanner(source, scanMoorTokens: useMoorExtensions);
|
||||||
final tokens = scanner.scanTokens();
|
final tokens = scanner.scanTokens();
|
||||||
|
@ -49,7 +54,8 @@ class SqlEngine {
|
||||||
/// Parses the [sql] statement into an AST-representation.
|
/// Parses the [sql] statement into an AST-representation.
|
||||||
ParseResult parse(String sql) {
|
ParseResult parse(String sql) {
|
||||||
final tokens = tokenize(sql);
|
final tokens = tokenize(sql);
|
||||||
final parser = Parser(tokens, useMoor: useMoorExtensions);
|
final tokensForParser = tokens.where((t) => !t.invisibleToParser).toList();
|
||||||
|
final parser = Parser(tokensForParser, useMoor: useMoorExtensions);
|
||||||
|
|
||||||
final stmt = parser.statement();
|
final stmt = parser.statement();
|
||||||
return ParseResult._(stmt, tokens, parser.errors, sql, null);
|
return ParseResult._(stmt, tokens, parser.errors, sql, null);
|
||||||
|
@ -62,7 +68,9 @@ class SqlEngine {
|
||||||
|
|
||||||
final autoComplete = AutoCompleteEngine();
|
final autoComplete = AutoCompleteEngine();
|
||||||
final tokens = tokenize(content);
|
final tokens = tokenize(content);
|
||||||
final parser = Parser(tokens, useMoor: true, autoComplete: autoComplete);
|
final tokensForParser = tokens.where((t) => !t.invisibleToParser).toList();
|
||||||
|
final parser =
|
||||||
|
Parser(tokensForParser, useMoor: true, autoComplete: autoComplete);
|
||||||
|
|
||||||
final moorFile = parser.moorFile();
|
final moorFile = parser.moorFile();
|
||||||
|
|
||||||
|
@ -134,7 +142,8 @@ class ParseResult {
|
||||||
/// The topmost node in the sql AST that was parsed.
|
/// The topmost node in the sql AST that was parsed.
|
||||||
final AstNode rootNode;
|
final AstNode rootNode;
|
||||||
|
|
||||||
/// The tokens that were scanned in the source file
|
/// The tokens that were scanned in the source file, including those that are
|
||||||
|
/// [Token.invisibleToParser].
|
||||||
final List<Token> tokens;
|
final List<Token> tokens;
|
||||||
|
|
||||||
/// A list of all errors that occurred during parsing. [ParsingError.toString]
|
/// A list of all errors that occurred during parsing. [ParsingError.toString]
|
||||||
|
|
|
@ -61,13 +61,22 @@ class Scanner {
|
||||||
_addToken(TokenType.plus);
|
_addToken(TokenType.plus);
|
||||||
break;
|
break;
|
||||||
case '-':
|
case '-':
|
||||||
_addToken(TokenType.minus);
|
if (_match('-')) {
|
||||||
|
_lineComment();
|
||||||
|
} else {
|
||||||
|
_addToken(TokenType.minus);
|
||||||
|
}
|
||||||
break;
|
break;
|
||||||
case '*':
|
case '*':
|
||||||
_addToken(TokenType.star);
|
_addToken(TokenType.star);
|
||||||
break;
|
break;
|
||||||
case '/':
|
case '/':
|
||||||
_addToken(TokenType.slash);
|
if (_match('*')) {
|
||||||
|
_cStyleComment();
|
||||||
|
} else {
|
||||||
|
_addToken(TokenType.slash);
|
||||||
|
}
|
||||||
|
|
||||||
break;
|
break;
|
||||||
case '%':
|
case '%':
|
||||||
_addToken(TokenType.percent);
|
_addToken(TokenType.percent);
|
||||||
|
@ -378,4 +387,37 @@ class Scanner {
|
||||||
tokens.add(InlineDartToken(_currentSpan));
|
tokens.add(InlineDartToken(_currentSpan));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Scans a line comment after the -- has already been read.
|
||||||
|
void _lineComment() {
|
||||||
|
final contentBuilder = StringBuffer();
|
||||||
|
while (_peek() != '\n' && !_isAtEnd) {
|
||||||
|
contentBuilder.write(_nextChar());
|
||||||
|
}
|
||||||
|
|
||||||
|
tokens.add(CommentToken(
|
||||||
|
CommentMode.line, contentBuilder.toString(), _currentSpan));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Scans a /* ... */ comment after the first /* has already been read.
|
||||||
|
/// Note that in sqlite, these comments don't have to be terminated - they
|
||||||
|
/// will be closed by the end of input without causing a parsing error.
|
||||||
|
void _cStyleComment() {
|
||||||
|
final contentBuilder = StringBuffer();
|
||||||
|
while (!_isAtEnd) {
|
||||||
|
if (_match('*')) {
|
||||||
|
if (!_isAtEnd && _match('/')) {
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
// write the * we otherwise forgot to write
|
||||||
|
contentBuilder.write('*');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
contentBuilder.write(_nextChar());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tokens.add(CommentToken(
|
||||||
|
CommentMode.cStyle, contentBuilder.toString(), _currentSpan));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -136,6 +136,7 @@ enum TokenType {
|
||||||
action,
|
action,
|
||||||
|
|
||||||
semicolon,
|
semicolon,
|
||||||
|
comment,
|
||||||
eof,
|
eof,
|
||||||
|
|
||||||
/// Moor specific token, used to declare a type converters
|
/// Moor specific token, used to declare a type converters
|
||||||
|
@ -252,6 +253,10 @@ const Map<String, TokenType> moorKeywords = {
|
||||||
class Token {
|
class Token {
|
||||||
final TokenType type;
|
final TokenType type;
|
||||||
|
|
||||||
|
/// Whether this token should be invisible to the parser. We use this for
|
||||||
|
/// comment tokens.
|
||||||
|
bool get invisibleToParser => false;
|
||||||
|
|
||||||
final FileSpan span;
|
final FileSpan span;
|
||||||
String get lexeme => span.text;
|
String get lexeme => span.text;
|
||||||
|
|
||||||
|
@ -348,6 +353,22 @@ class KeywordToken extends Token {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
enum CommentMode { line, cStyle }
|
||||||
|
|
||||||
|
/// A comment, either started with -- or with /*.
|
||||||
|
class CommentToken extends Token {
|
||||||
|
final CommentMode mode;
|
||||||
|
|
||||||
|
/// The content of this comment, excluding the "--", "/*", "*/".
|
||||||
|
final String content;
|
||||||
|
|
||||||
|
@override
|
||||||
|
final bool invisibleToParser = true;
|
||||||
|
|
||||||
|
CommentToken(this.mode, this.content, FileSpan span)
|
||||||
|
: super(TokenType.comment, span);
|
||||||
|
}
|
||||||
|
|
||||||
class TokenizerError {
|
class TokenizerError {
|
||||||
final String message;
|
final String message;
|
||||||
final SourceLocation location;
|
final SourceLocation location;
|
||||||
|
|
|
@ -0,0 +1,67 @@
|
||||||
|
import 'package:sqlparser/sqlparser.dart';
|
||||||
|
import 'package:sqlparser/src/utils/ast_equality.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
const content = r'''
|
||||||
|
import 'other.dart';
|
||||||
|
import 'another.moor';
|
||||||
|
|
||||||
|
CREATE TABLE tbl (
|
||||||
|
id INT NOT NULL PRIMARY KEY AUTOINCREMENT,
|
||||||
|
-- this is a single-line comment
|
||||||
|
place VARCHAR REFERENCES other(location)
|
||||||
|
)
|
||||||
|
|
||||||
|
all: SELECT /* COUNT(*), */ * FROM tbl WHERE $predicate;
|
||||||
|
''';
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
test('parses moor files', () {
|
||||||
|
final parsed = SqlEngine(useMoorExtensions: true).parseMoorFile(content);
|
||||||
|
final file = parsed.rootNode;
|
||||||
|
|
||||||
|
enforceEqual(
|
||||||
|
file,
|
||||||
|
MoorFile([
|
||||||
|
ImportStatement('other.dart'),
|
||||||
|
ImportStatement('another.moor'),
|
||||||
|
CreateTableStatement(
|
||||||
|
tableName: 'tbl',
|
||||||
|
columns: [
|
||||||
|
ColumnDefinition(
|
||||||
|
columnName: 'id',
|
||||||
|
typeName: 'INT',
|
||||||
|
constraints: [
|
||||||
|
NotNull(null),
|
||||||
|
PrimaryKeyColumn(null, autoIncrement: true),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
ColumnDefinition(
|
||||||
|
columnName: 'place',
|
||||||
|
typeName: 'VARCHAR',
|
||||||
|
constraints: [
|
||||||
|
ForeignKeyColumnConstraint(
|
||||||
|
null,
|
||||||
|
ForeignKeyClause(
|
||||||
|
foreignTable: TableReference('other', null),
|
||||||
|
columnNames: [
|
||||||
|
Reference(columnName: 'location'),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
DeclaredStatement(
|
||||||
|
'all',
|
||||||
|
SelectStatement(
|
||||||
|
columns: [StarResultColumn(null)],
|
||||||
|
from: [TableReference('tbl', null)],
|
||||||
|
where: DartExpressionPlaceholder(name: 'predicate'),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
|
@ -0,0 +1,35 @@
|
||||||
|
import 'package:sqlparser/src/reader/tokenizer/scanner.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
import '../parser/utils.dart';
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
test('scanns comments', () {
|
||||||
|
const sql = r'''
|
||||||
|
--line
|
||||||
|
-- line
|
||||||
|
/*c*/
|
||||||
|
/*multi
|
||||||
|
line */
|
||||||
|
/* not terminated''';
|
||||||
|
|
||||||
|
// using whereType instead of cast because of the invisible eof token
|
||||||
|
final tokens = Scanner(sql).scanTokens().whereType<CommentToken>();
|
||||||
|
|
||||||
|
expect(tokens.map((t) => t.mode), [
|
||||||
|
CommentMode.line,
|
||||||
|
CommentMode.line,
|
||||||
|
CommentMode.cStyle,
|
||||||
|
CommentMode.cStyle,
|
||||||
|
CommentMode.cStyle,
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(tokens.map((t) => t.content), [
|
||||||
|
'line',
|
||||||
|
' line',
|
||||||
|
'c',
|
||||||
|
'multi\n line ',
|
||||||
|
' not terminated',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
}
|
Loading…
Reference in New Issue