diff --git a/drift_dev/lib/src/analyzer/moor/create_table_reader.dart b/drift_dev/lib/src/analyzer/moor/create_table_reader.dart index 53442bce..8d0820fe 100644 --- a/drift_dev/lib/src/analyzer/moor/create_table_reader.dart +++ b/drift_dev/lib/src/analyzer/moor/create_table_reader.dart @@ -34,7 +34,8 @@ class CreateTableReader { Table table; try { table = _schemaReader.read(stmt); - } catch (e) { + } catch (e, s) { + print(s); step.reportError(ErrorInMoorFile( span: stmt.tableNameToken!.span, message: 'Could not extract schema information for this table: $e', diff --git a/drift_dev/lib/src/analyzer/sql_queries/type_mapping.dart b/drift_dev/lib/src/analyzer/sql_queries/type_mapping.dart index 063dffb6..833d41d2 100644 --- a/drift_dev/lib/src/analyzer/sql_queries/type_mapping.dart +++ b/drift_dev/lib/src/analyzer/sql_queries/type_mapping.dart @@ -304,8 +304,7 @@ class TypeMapper { }, ); - final availableResults = - placeholder.scope.allOf(); + final availableResults = placeholder.statementScope.allAvailableResultSets; final availableMoorResults = []; for (final available in availableResults) { final aliasedResultSet = available.resultSet.resultSet; diff --git a/sqlparser/CHANGELOG.md b/sqlparser/CHANGELOG.md index 76fb9d85..849b4414 100644 --- a/sqlparser/CHANGELOG.md +++ b/sqlparser/CHANGELOG.md @@ -1,5 +1,6 @@ -## 0.21.1-dev +## 0.22.0-dev +- Refactor how tables and columns are resolved internally. - Lint for `DISTINCT` misuse in aggregate function calls. ## 0.21.0 diff --git a/sqlparser/lib/src/analysis/analysis.dart b/sqlparser/lib/src/analysis/analysis.dart index c44a0212..52d680bb 100644 --- a/sqlparser/lib/src/analysis/analysis.dart +++ b/sqlparser/lib/src/analysis/analysis.dart @@ -7,6 +7,8 @@ import 'package:source_span/source_span.dart'; import 'package:sqlparser/sqlparser.dart' hide ExpandParameters; import 'package:sqlparser/src/utils/meta.dart'; +import '../../utils/case_insensitive_map.dart'; + export 'types/data.dart'; export 'types/types.dart' show TypeInferenceResults; diff --git a/sqlparser/lib/src/analysis/context.dart b/sqlparser/lib/src/analysis/context.dart index 639eb8b0..bccd5434 100644 --- a/sqlparser/lib/src/analysis/context.dart +++ b/sqlparser/lib/src/analysis/context.dart @@ -13,6 +13,11 @@ class AnalysisContext { /// The raw sql statement that was used to construct this [AnalysisContext]. final String sql; + /// The root scope used when analyzing SQL statements. + /// + /// This contains information about known tables and modules. + final RootScope rootScope; + /// Additional information about variables in this context, passed from the /// outside. final AnalyzeStatementOptions stmtOptions; @@ -34,7 +39,7 @@ class AnalysisContext { late final TypeInferenceResults types2; /// Constructs a new analysis context from the AST and the source sql. - AnalysisContext(this.root, this.sql, this.engineOptions, + AnalysisContext(this.root, this.sql, this.rootScope, this.engineOptions, {AnalyzeStatementOptions? stmtOptions, required this.schemaSupport}) : stmtOptions = stmtOptions ?? const AnalyzeStatementOptions(); diff --git a/sqlparser/lib/src/analysis/schema/from_create_table.dart b/sqlparser/lib/src/analysis/schema/from_create_table.dart index 028a0276..581b6d29 100644 --- a/sqlparser/lib/src/analysis/schema/from_create_table.dart +++ b/sqlparser/lib/src/analysis/schema/from_create_table.dart @@ -16,7 +16,7 @@ class SchemaFromCreateTable { if (stmt is CreateTableStatement) { return _readCreateTable(stmt); } else if (stmt is CreateVirtualTableStatement) { - final module = stmt.scope.resolve(stmt.moduleName); + final module = stmt.scope.rootScope.knownModules[stmt.moduleName]; if (module == null) { throw CantReadSchemaException('Unknown module "${stmt.moduleName}", ' diff --git a/sqlparser/lib/src/analysis/schema/references.dart b/sqlparser/lib/src/analysis/schema/references.dart index b9bcab81..2b626b3d 100644 --- a/sqlparser/lib/src/analysis/schema/references.dart +++ b/sqlparser/lib/src/analysis/schema/references.dart @@ -20,126 +20,337 @@ mixin Referencable { bool get visibleToChildren => false; } -/// Class which keeps track of references for tables, columns and functions in a -/// query. -class ReferenceScope { - final ReferenceScope? parent; - final ReferenceScope? root; +/// A class managing which tables and columns are visible to which AST nodes. +abstract class ReferenceScope { + RootScope get rootScope; - /// Whether the [availableColumns] of the [parent] are available in this scope - /// as well. - final bool inheritAvailableColumns; + /// The list of column to which a `*` would expand to. + /// + /// This is not necessary the same list of columns that could be resolved + /// through [resolveUnqualifiedReference]. For subquery expressions, columns + /// in parent scopes may be referenced without a qualified, but they don't + /// appear in a `*` expansion for the subquery. + List? get expansionOfStarColumn => null; - /// Gets the effective root scope. If no [root] scope has been set, this - /// scope is assumed to be the root scope. - ReferenceScope get effectiveRoot => root ?? this; + /// Attempts to find a result set that has been added to this scope, for + /// instance because it was introduced in a `FROM` clause. + /// + /// This is useful to resolve qualified references (e.g. to resolve `foo.bar` + /// the resolver would call [resolveResultSet]("foo") and then look up the + /// `bar` column in that result set). + ResultSetAvailableInStatement? resolveResultSet(String name) => null; - final Map> _references = {}; + /// Adds an added result set to this scope. + /// + /// This operation is not supported for all kinds of scopes, a [StateError] + /// is thrown for invalid scopes. + void addResolvedResultSet( + String? name, ResultSetAvailableInStatement resultSet) { + throw StateError('Result set cannot be added in this scope: $this'); + } - List? _availableColumns; + /// Registers a [ResultSetAvailableInStatement] to a [TableAlias] for the + /// given [resultSet]. + /// + /// Like [addResolvedResultSet], this operation is not supported on all + /// scopes. + void addAlias(AstNode origin, ResultSet resultSet, String alias) { + final createdAlias = TableAlias(resultSet, alias); + addResolvedResultSet( + alias, ResultSetAvailableInStatement(origin, createdAlias)); + } - /// All columns that would be available in this scope. Can be used to resolve - /// a '*' expression for function calls or result columns - List get availableColumns { - if (_availableColumns != null) return _availableColumns!; - if (inheritAvailableColumns) return parent!.availableColumns; + /// Attempts to find a result set that _can_ be added to a scope. + /// + /// This is used to resolve table references. Usually, after a result set to + /// add has been resolve,d a [ResultSetAvailableInStatement] is added to the + /// scope and [resolveResultSet] will find that afterwards. + ResultSet? resolveResultSetToAdd(String name) => rootScope.knownTables[name]; + + /// Attempts to resolve an unqualified reference from a [columnName]. + /// + /// In sqlite, an `ORDER BY` column may refer to aliases of result columns + /// in the current statement: `SELECT foo AS bar FROM tbl ORDER BY bar` is + /// legal, but `SELECT foo AS bar FROM tbl WHERE bar < 10` is not. To control + /// whether result columns may be resolved, the [allowReferenceToResultColumn] + /// flag can be enabled. + /// + /// If an empty list is returned, the reference couldn't be resolved. If the + /// returned list contains more than one column, the lookup is ambigious. + List resolveUnqualifiedReference(String columnName, + {bool allowReferenceToResultColumn = false}) => + const []; +} + +/// The root scope created by the SQL engine to analyze a statement. +/// +/// This contains known tables (or views) and modules to look up. +class RootScope extends ReferenceScope { + @override + RootScope get rootScope => this; + + /// All tables (or views, or other result sets) that are known in the current + /// schema. + /// + /// [resolveResultSetToAdd] will query these tables by default. + final Map knownTables = CaseInsensitiveMap(); + + /// Known modules that are registered for this statement. + /// + /// This is used to resolve `CREATE VIRTUAL TABLE` statements. + final Map knownModules = CaseInsensitiveMap(); +} + +/// A scope used by statements. +/// +/// Tables added from `FROM` clauses are added to [resultSets], CTEs are added +/// to [additionalKnownTables]. +/// +/// This is the scope most commonly used, but specific nodes may be attached to +/// a different scope in case they have limited visibility. For instance, +/// - foreign key clauses are wrapped in a [SingleTableReferenceScope] because +/// they can't see unqualified columns of the overal scope. +/// - subquery expressions can see parent tables and columns, but their columns +/// aren't visible in the parent statement. This is implemented by wrapping +/// them in a [StatementScope] as well. +/// - subqueries appearing in a `FROM` clause _can't_ see outer columns and +/// tables. These statements are also wrapped in a [StatementScope], but a +/// [SubqueryInFromScope] is insertted as an intermediatet scope to prevent +/// the inner scope from seeing the outer columns. + +class StatementScope extends ReferenceScope { + final ReferenceScope parent; + + /// Additional tables (that haven't necessarily been added in a `FROM` clause + /// that are only visible in this scope). + /// + /// This is commonly used for common table expressions, e.g a `WITH foo AS + /// (...)` would add a result set `foo` into the [additionalKnownTables] of + /// the overall statement, because `foo` can now be selected. + final Map additionalKnownTables = CaseInsensitiveMap(); + + /// Result sets that were added through a `FROM` clause and are now available + /// in this scope. + /// + /// The [ResultSetAvailableInStatement] contains information about the AST + /// node causing this statement to be available. + final Map resultSets = + CaseInsensitiveMap(); + + /// For select statements, additional columns available under a name because + /// there were added after the `SELECT`. + /// + /// This is used to resolve unqualified references by `ORDER BY` clauses. + final List namedResultColumns = []; + + final Map windowDeclarations = + CaseInsensitiveMap(); + + /// All columns that a (unqualified) `*` in a select statement or function + /// call argument would expand to. + @override + List? expansionOfStarColumn; + + StatementScope(this.parent); + + StatementScope? get parentStatementScope { + final parent = this.parent; + if (parent is StatementScope) { + return parent; + } else if (parent is MiscStatementSubScope) { + return parent.parent; + } else { + return null; + } + } + + /// All result sets available in this and parent scopes. + Iterable get allAvailableResultSets { + final here = resultSets.values; + final parent = parentStatementScope; + return parent != null + ? here.followedBy(parent.allAvailableResultSets) + : here; + } + + @override + RootScope get rootScope => parent.rootScope; + + @override + void addAlias(AstNode origin, ResultSet resultSet, String alias) { + final createdAlias = TableAlias(resultSet, alias); + additionalKnownTables[alias] = createdAlias; + resultSets[alias] = ResultSetAvailableInStatement(origin, createdAlias); + } + + @override + ResultSetAvailableInStatement? resolveResultSet(String name) { + return resultSets[name] ?? parentStatementScope?.resolveResultSet(name); + } + + @override + void addResolvedResultSet( + String? name, ResultSetAvailableInStatement resultSet) { + resultSets[name] = resultSet; + } + + @override + ResultSet? resolveResultSetToAdd(String name) { + return additionalKnownTables[name] ?? + parentStatementScope?.resolveResultSetToAdd(name) ?? + rootScope.knownTables[name]; + } + + @override + List resolveUnqualifiedReference(String columnName, + {bool allowReferenceToResultColumn = false}) { + if (allowReferenceToResultColumn) { + final foundColumn = namedResultColumns.firstWhereOrNull( + (c) => c.name.toLowerCase() == columnName.toLowerCase()); + if (foundColumn != null) { + return [foundColumn]; + } + } + + StatementScope? currentScope = this; + + // Search scopes for a matching column in an added result set. If a column + // reference is found in a closer scope, it takes precedence over outer + // scopes. However, it's an error if two columns with the same name are + // found in the same scope. + while (currentScope != null) { + final available = currentScope.resultSets.values; + final sourceColumns = {}; + final availableColumns = []; + + for (final availableSource in available) { + final resolvedColumns = + availableSource.resultSet.resultSet?.resolvedColumns; + if (resolvedColumns == null) continue; + + for (final column in resolvedColumns) { + if (column.name.toLowerCase() == columnName.toLowerCase() && + sourceColumns.add(column)) { + availableColumns.add(AvailableColumn(column, availableSource)); + } + } + } + + if (availableColumns.isEmpty) { + currentScope = currentScope.parentStatementScope; + if (currentScope == null) { + // Reached the outermost scope without finding a reference target. + return const []; + } + continue; + } else { + return availableColumns; + } + } return const []; } - set availableColumns(List? value) { - // guard against lists of subtype of column - if (value != null) { - _availableColumns = [...value]; + factory StatementScope.forStatement(RootScope root, Statement statement) { + return StatementScope(statement.optionalScope ?? root); + } + + static StatementScope cast(ReferenceScope other) { + if (other is StatementScope) { + return other; + } else if (other is MiscStatementSubScope) { + return other.parent; } else { - _availableColumns = null; + throw ArgumentError.value( + other, 'other', 'Not resolvable to a statement scope'); + } + } +} + +/// A special intermediate scope used for subqueries appearing in a `FROM` +/// clause so that the subquery can't see outer columns and tables being added. +class SubqueryInFromScope extends ReferenceScope { + final StatementScope enclosingStatement; + + SubqueryInFromScope(this.enclosingStatement); + + @override + RootScope get rootScope => enclosingStatement.rootScope; +} + +/// A rarely used sub-scope for AST nodes that belong to a statement, but may +/// have access to more result sets. +/// +/// For instance, the body of an `ON CONFLICT DO UPDATE`-clause may refer to a +/// table alias `excluded` to get access to a conflicting table. +class MiscStatementSubScope extends ReferenceScope { + final StatementScope parent; + + final Map additionalResultSets = + CaseInsensitiveMap(); + + MiscStatementSubScope(this.parent); + + @override + RootScope get rootScope => parent.rootScope; + + @override + ResultSetAvailableInStatement? resolveResultSet(String name) { + return additionalResultSets[name] ?? parent.resolveResultSet(name); + } + + @override + void addResolvedResultSet( + String? name, ResultSetAvailableInStatement resultSet) { + additionalResultSets[name] = resultSet; + } + + @override + List resolveUnqualifiedReference(String columnName, + {bool allowReferenceToResultColumn = false}) { + return parent.resolveUnqualifiedReference(columnName); + } +} + +/// A reference scope that only allows a single added result set. +/// +/// This is used for e.g. foreign key clauses (`REFERENCES table (a, b, c)`), +/// where `a`, `b` and `c` can only refer to `table`. +class SingleTableReferenceScope extends ReferenceScope { + final ReferenceScope parent; + + String? addedTableName; + ResultSetAvailableInStatement? addedTable; + + SingleTableReferenceScope(this.parent); + + @override + RootScope get rootScope => parent.rootScope; + + @override + ResultSetAvailableInStatement? resolveResultSet(String name) { + if (name == addedTableName) { + return addedTable; + } else { + return null; + } + } + + @override + void addResolvedResultSet( + String? name, ResultSetAvailableInStatement resultSet) { + addedTableName = null; + addedTable = null; + } + + @override + List resolveUnqualifiedReference(String columnName, + {bool allowReferenceToResultColumn = false}) { + final column = addedTable?.resultSet.resultSet?.findColumn(columnName); + if (column != null) { + return [AvailableColumn(column, addedTable!)]; + } else { + return const []; } } - - ReferenceScope(this.parent, - {this.root, this.inheritAvailableColumns = false}); - - void addAvailableColumn(Column column) { - // make sure _availableColumns is resolved and mutable - final ownColumns = _availableColumns ??= [...availableColumns]; - ownColumns.add(column); - } - - ReferenceScope createChild({bool? inheritAvailableColumns}) { - // wonder why we're creating a linked list of reference scopes instead of - // just passing down a copy of [_references]? In sql, some variables can be - // used before they're defined, even in child scopes: - // SELECT *, (SELECT * FROM table2 WHERE id = t1.a) FROM table2 t1 - return ReferenceScope( - this, - root: effectiveRoot, - inheritAvailableColumns: inheritAvailableColumns ?? false, - ); - } - - ReferenceScope createSibling() { - return parent!.createChild(); - } - - /// Registers something that can be referenced in this and child scopes. - void register(String identifier, Referencable ref) { - _references.putIfAbsent(identifier.toUpperCase(), () => []).add(ref); - } - - /// Registers both a [TableAlias] and a [ResultSetAvailableInStatement] so - /// that the alias can be used in expressions without being selected. - void registerUsableAlias(AstNode origin, ResultSet resultSet, String alias) { - final createdAlias = TableAlias(resultSet, alias); - register(alias, createdAlias); - register(alias, ResultSetAvailableInStatement(origin, createdAlias)); - } - - /// Resolves to a [Referencable] with the given [name] and of the type [T]. - /// If the reference couldn't be found, null is returned and [orElse] will be - /// called. - T? resolve(String name, {Function()? orElse}) { - ReferenceScope? scope = this; - var isAtParent = false; - final upper = name.toUpperCase(); - - while (scope != null) { - if (scope._references.containsKey(upper)) { - final candidates = scope._references[upper]!; - final resolved = candidates.whereType().where((x) { - return x.visibleToChildren || !isAtParent; - }); - if (resolved.isNotEmpty) { - return resolved.first; - } - } - - scope = scope.parent; - isAtParent = true; - } - - if (orElse != null) orElse(); - return null; // not found in any parent scope - } - - /// Returns everything that is in scope and a subtype of [T]. - List allOf() { - ReferenceScope? scope = this; - var isInCurrentScope = true; - final collected = []; - - while (scope != null) { - var foundValues = - scope._references.values.expand((list) => list).whereType(); - - if (!isInCurrentScope) { - foundValues = - foundValues.where((element) => element.visibleToChildren).cast(); - } - - collected.addAll(foundValues); - scope = scope.parent; - isInCurrentScope = false; - } - return collected; - } } diff --git a/sqlparser/lib/src/analysis/schema/table.dart b/sqlparser/lib/src/analysis/schema/table.dart index 801a4d4b..9b1fe0d2 100644 --- a/sqlparser/lib/src/analysis/schema/table.dart +++ b/sqlparser/lib/src/analysis/schema/table.dart @@ -80,6 +80,9 @@ class Table extends NamedResultSet with HasMetaMixin implements HumanReadable { String humanReadableDescription() { return name; } + + @override + String toString() => 'Table $name'; } class TableAlias extends NamedResultSet implements HumanReadable { diff --git a/sqlparser/lib/src/analysis/steps/column_resolver.dart b/sqlparser/lib/src/analysis/steps/column_resolver.dart index affcd645..3f70127d 100644 --- a/sqlparser/lib/src/analysis/steps/column_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/column_resolver.dart @@ -57,7 +57,7 @@ class ColumnResolver extends RecursiveVisitor { if (table != null) { // add "excluded" table qualifier that referring to the row that would // have been inserted had the uniqueness constraint not been violated. - e.scope.registerUsableAlias(e, table, 'excluded'); + e.scope.addAlias(e, table, 'excluded'); } visitChildren(e, arg); @@ -80,7 +80,7 @@ class ColumnResolver extends RecursiveVisitor { final from = e.from; if (from != null) _handle(from, availableColumns); - e.scope.availableColumns = availableColumns; + e.statementScope.expansionOfStarColumn = availableColumns; for (final child in e.childNodes) { // Visit remaining children if (child != e.table && child != e.from) visit(child, arg); @@ -92,7 +92,7 @@ class ColumnResolver extends RecursiveVisitor { ResultSet? _addIfResolved(AstNode node, TableReference ref) { final table = _resolveTableReference(ref); if (table != null) { - node.scope.availableColumns = table.resolvedColumns; + node.statementScope.expansionOfStarColumn = table.resolvedColumns; } return table; @@ -135,7 +135,7 @@ class ColumnResolver extends RecursiveVisitor { final clause = stmt.returning; if (clause == null) return; - final columns = _resolveColumns(stmt.scope, clause.columns, + final columns = _resolveColumns(stmt.statementScope, clause.columns, columnsForStar: mainTable?.resolvedColumns); stmt.returnedResultSet = CustomResultSet(columns); } @@ -149,16 +149,16 @@ class ColumnResolver extends RecursiveVisitor { return; } - final scope = e.scope; + final scope = e.statementScope; // Add columns of the target table for when and update of clauses - scope.availableColumns = table.resolvedColumns; + scope.expansionOfStarColumn = table.resolvedColumns; if (e.target.introducesNew) { - scope.registerUsableAlias(e, table, 'new'); + scope.addAlias(e, table, 'new'); } if (e.target.introducesOld) { - scope.registerUsableAlias(e, table, 'old'); + scope.addAlias(e, table, 'old'); } visitChildren(e, arg); @@ -236,16 +236,16 @@ class ColumnResolver extends RecursiveVisitor { _handle(s.from!, availableColumns); } - final scope = s.scope; - scope.availableColumns = availableColumns; + final scope = s.statementScope; + scope.expansionOfStarColumn = availableColumns; s.resolvedColumns = _resolveColumns(scope, s.columns); } - List _resolveColumns(ReferenceScope scope, List columns, + List _resolveColumns(StatementScope scope, List columns, {List? columnsForStar}) { final usedColumns = []; - final availableColumns = [...scope.availableColumns]; + final availableColumns = [...?scope.expansionOfStarColumn]; // a select statement can include everything from its sub queries as a // result, but also expressions that appear as result columns @@ -254,15 +254,15 @@ class ColumnResolver extends RecursiveVisitor { Iterable? visibleColumnsForStar; if (resultColumn.tableName != null) { - final tableResolver = scope.resolve( - resultColumn.tableName!, orElse: () { + final tableResolver = scope.resolveResultSet(resultColumn.tableName!); + if (tableResolver == null) { context.reportError(AnalysisError( type: AnalysisErrorType.referencedUnknownTable, message: 'Unknown table: ${resultColumn.tableName}', relevantNode: resultColumn, )); - }); - if (tableResolver == null) continue; + continue; + } visibleColumnsForStar = tableResolver.resultSet.resultSet?.resolvedColumns?.map( @@ -274,7 +274,7 @@ class ColumnResolver extends RecursiveVisitor { // Star columns can't be used without a table (e.g. `SELECT *` is // not allowed) - if (scope.allOf().isEmpty) { + if (scope.resultSets.isEmpty) { context.reportError(AnalysisError( type: AnalysisErrorType.starColumnWithoutTable, message: "Can't use * when no tables have been added", @@ -306,20 +306,22 @@ class ColumnResolver extends RecursiveVisitor { final name = resultColumn.as; if (!availableColumns.any((c) => c.name == name)) { availableColumns.add(column); - scope.addAvailableColumn(column); + scope.namedResultColumns.add(column); } } } else if (resultColumn is NestedStarResultColumn) { - final target = scope.resolve( - resultColumn.tableName, orElse: () { + final target = scope.resolveResultSet(resultColumn.tableName); + + if (target == null) { context.reportError(AnalysisError( type: AnalysisErrorType.referencedUnknownTable, message: 'Unknown table: ${resultColumn.tableName}', relevantNode: resultColumn, )); - }); + continue; + } - if (target != null) resultColumn.resultSet = target.resultSet.resultSet; + resultColumn.resultSet = target.resultSet.resultSet; } else if (resultColumn is NestedQueryColumn) { _resolveSelect(resultColumn.select); } @@ -410,9 +412,8 @@ class ColumnResolver extends RecursiveVisitor { // Try resolving to a top-level table in the schema and to a result set that // may have been added to the table - final resolvedInSchema = scope.resolve(r.tableName); - final resolvedInQuery = - scope.resolve(r.tableName); + final resolvedInSchema = scope.resolveResultSetToAdd(r.tableName); + final resolvedInQuery = scope.resolveResultSet(r.tableName); final createdName = r.as; // Prefer using a table that has already been added if this isn't the @@ -428,26 +429,27 @@ class ColumnResolver extends RecursiveVisitor { ? TableAlias(resolvedInSchema, createdName) : resolvedInSchema; } else { - final available = scope - .allOf() - .followedBy(scope - .allOf() - .map((added) => added.resultSet)) - .where((e) => e.resultSet != null) - .map((t) { - final resultSet = t.resultSet; - if (resultSet is HumanReadable) { - return (resultSet as HumanReadable).humanReadableDescription(); - } + Iterable? available; - return t.toString(); - }); + if (scope is StatementScope) { + available = StatementScope.cast(scope) + .allAvailableResultSets + .where((e) => e.resultSet.resultSet != null) + .map((t) { + final resultSet = t.resultSet.resultSet; + if (resultSet is HumanReadable) { + return (resultSet as HumanReadable).humanReadableDescription(); + } + + return t.toString(); + }); + } context.reportError(UnresolvedReferenceError( type: AnalysisErrorType.referencedUnknownTable, relevantNode: r, reference: r.tableName, - available: available, + available: available ?? const Iterable.empty(), )); } diff --git a/sqlparser/lib/src/analysis/steps/prepare_ast.dart b/sqlparser/lib/src/analysis/steps/prepare_ast.dart index 4437a1a8..3059bbd2 100644 --- a/sqlparser/lib/src/analysis/steps/prepare_ast.dart +++ b/sqlparser/lib/src/analysis/steps/prepare_ast.dart @@ -21,16 +21,15 @@ class AstPreparingVisitor extends RecursiveVisitor { @override void visitCreateTableStatement(CreateTableStatement e, void arg) { - final scope = e.scope = e.scope.createChild(); - final registeredTable = scope.resolve(e.tableName) as Table?; + final scope = e.scope = StatementScope.forStatement(context.rootScope, e); + final knownTable = context.rootScope.knownTables[e.tableName]; // This is used so that tables can refer to their own columns. Code using // tables would first register the table and then run analysis again. - if (registeredTable != null) { - scope.availableColumns = registeredTable.resolvedColumns; - for (final column in registeredTable.resolvedColumns) { - scope.register(column.name, column); - } + if (knownTable is Table) { + scope + ..expansionOfStarColumn = knownTable.resolvedColumns + ..resultSets[null] = ResultSetAvailableInStatement(e, knownTable); } visitChildren(e, arg); @@ -38,7 +37,7 @@ class AstPreparingVisitor extends RecursiveVisitor { @override void visitCreateViewStatement(CreateViewStatement e, void arg) { - e.scope = e.scope.createChild(); + e.scope = StatementScope.forStatement(context.rootScope, e); visitChildren(e, arg); } @@ -54,20 +53,21 @@ class AstPreparingVisitor extends RecursiveVisitor { // (SELECT * FROM demo i WHERE i.id = d1.id) d2;" // it won't work. final isInFROM = e.parent is Queryable; - final scope = e.scope; + StatementScope scope; if (isInFROM) { - final surroundingSelect = - e.parents.firstWhere((node) => node is HasFrom).scope; - final forked = surroundingSelect.createSibling(); - e.scope = forked; + final surroundingSelect = e.parents + .firstWhere((node) => node is HasFrom) + .scope as StatementScope; + scope = StatementScope(SubqueryInFromScope(surroundingSelect)); } else { - final forked = scope.createChild(); - e.scope = forked; + scope = StatementScope.forStatement(context.rootScope, e); } + e.scope = scope; + for (final windowDecl in e.windowDeclarations) { - e.scope.register(windowDecl.name, windowDecl); + scope.windowDeclarations[windowDecl.name] = windowDecl; } // only the last statement in a compound select statement may have an order @@ -107,18 +107,6 @@ class AstPreparingVisitor extends RecursiveVisitor { visitChildren(e, arg); } - @override - void visitResultColumn(ResultColumn e, void arg) { - if (e is StarResultColumn) { - // doesn't need special treatment, star expressions can't be referenced - } else if (e is ExpressionResultColumn) { - if (e.as != null) { - e.scope.register(e.as!, e); - } - } - visitChildren(e, arg); - } - @override void defaultQueryable(Queryable e, void arg) { final scope = e.scope; @@ -128,14 +116,12 @@ class AstPreparingVisitor extends RecursiveVisitor { final added = ResultSetAvailableInStatement(table, table); table.availableResultSet = added; - scope.register(table.as ?? table.tableName, added); + scope.addResolvedResultSet(table.as ?? table.tableName, added); }, isSelect: (select) { - if (select.as != null) { - final added = ResultSetAvailableInStatement(select, select.statement); - select.availableResultSet = added; - scope.register(select.as!, added); - } + final added = ResultSetAvailableInStatement(select, select.statement); + select.availableResultSet = added; + scope.addResolvedResultSet(select.as, added); }, isJoin: (join) { // the join can contain multiple tables. Luckily for us, all of them are @@ -145,7 +131,7 @@ class AstPreparingVisitor extends RecursiveVisitor { isTableFunction: (function) { final added = ResultSetAvailableInStatement(function, function); function.availableResultSet = added; - scope.register(function.as ?? function.name, added); + scope.addResolvedResultSet(function.as ?? function.name, added); }, ); @@ -154,7 +140,13 @@ class AstPreparingVisitor extends RecursiveVisitor { @override void visitCommonTableExpression(CommonTableExpression e, void arg) { - e.scope.register(e.cteTableName, e); + StatementScope.cast(e.scope).additionalKnownTables[e.cteTableName] = e; + visitChildren(e, arg); + } + + @override + void visitForeignKeyClause(ForeignKeyClause e, void arg) { + e.scope = SingleTableReferenceScope(e.scope); visitChildren(e, arg); } @@ -203,16 +195,11 @@ class AstPreparingVisitor extends RecursiveVisitor { } } - void _forkScope(AstNode node, {bool? inheritAvailableColumns}) { - node.scope = node.scope - .createChild(inheritAvailableColumns: inheritAvailableColumns); - } - @override void defaultNode(AstNode e, void arg) { // hack to fork scopes on statements (selects are handled above) if (e is Statement && e is! SelectStatement) { - _forkScope(e); + e.scope = StatementScope.forStatement(context.rootScope, e); } visitChildren(e, arg); @@ -244,7 +231,7 @@ class AstPreparingVisitor extends RecursiveVisitor { // "excluded" can be referred. Setting that row happens in the column // resolver if (e.action is DoUpdate) { - _forkScope(e.action, inheritAvailableColumns: true); + e.action.scope = MiscStatementSubScope(e.scope as StatementScope); } visitChildren(e, null); @@ -257,7 +244,7 @@ class AstPreparingVisitor extends RecursiveVisitor { // create a new scope for the nested query to differentiate between // references that can be resolved in the nested query and references // which require data from the parent query - e.select.scope = e.scope.createChild(); + e.select.scope = MiscStatementSubScope(e.scope as StatementScope); AstPreparingVisitor(context: context).start(e.select); } else { super.visitDriftSpecificNode(e, arg); diff --git a/sqlparser/lib/src/analysis/steps/reference_resolver.dart b/sqlparser/lib/src/analysis/steps/reference_resolver.dart index bf8233bc..c45e3d74 100644 --- a/sqlparser/lib/src/analysis/steps/reference_resolver.dart +++ b/sqlparser/lib/src/analysis/steps/reference_resolver.dart @@ -43,8 +43,7 @@ class ReferenceResolver extends RecursiveVisitor { if (e.entityName != null) { // first find the referenced table or view, // then use the column on that table or view. - final entityResolver = - scope.resolve(e.entityName!); + final entityResolver = scope.resolveResultSet(e.entityName!); final resultSet = entityResolver?.resultSet.resultSet; if (resultSet == null) { @@ -68,15 +67,19 @@ class ReferenceResolver extends RecursiveVisitor { } else { // find any column with the referenced name. // todo special case for USING (...) in joins? - final columns = - scope.availableColumns.where((c) => c.name == e.columnName).toSet(); + final found = scope.resolveUnqualifiedReference( + e.columnName, + // According to https://www.sqlite.org/lang_select.html#the_order_by_clause, + // a simple reference in an ordering term can refer to an output column. + allowReferenceToResultColumn: e.parent is OrderingTerm, + ); - if (columns.isEmpty) { + if (found.isEmpty) { _reportUnknownColumnError(e); } else { - if (columns.length > 1) { + if (found.length > 1) { final description = - columns.map((c) => c.humanReadableDescription()).join(', '); + found.map((c) => c.humanReadableDescription()).join(', '); context.reportError(AnalysisError( type: AnalysisErrorType.ambiguousReference, @@ -85,7 +88,7 @@ class ReferenceResolver extends RecursiveVisitor { )); } - e.resolved = columns.first; + e.resolved = found.first; } } @@ -108,23 +111,25 @@ class ReferenceResolver extends RecursiveVisitor { @override void visitWindowFunctionInvocation(WindowFunctionInvocation e, void arg) { if (e.windowName != null && e.resolved == null) { - final resolved = e.scope.resolve(e.windowName!); - e.resolved = resolved; + e.resolved = + StatementScope.cast(e.scope).windowDeclarations[e.windowName!]; } visitChildren(e, arg); } void _reportUnknownColumnError(Reference e, {Iterable? columns}) { - columns ??= e.scope.availableColumns; - final columnNames = e.scope.availableColumns - .map((c) => c.humanReadableDescription()) - .join(', '); + final msg = StringBuffer('Unknown column.'); + if (columns != null) { + final columnNames = + columns.map((c) => c.humanReadableDescription()).join(', '); + msg.write(' Thesecolumns are available: $columnNames'); + } context.reportError(AnalysisError( type: AnalysisErrorType.referencedUnknownColumn, relevantNode: e, - message: 'Unknown column. These columns are available: $columnNames', + message: msg.toString(), )); } diff --git a/sqlparser/lib/src/analysis/types/resolving_visitor.dart b/sqlparser/lib/src/analysis/types/resolving_visitor.dart index d520f31c..d059a74d 100644 --- a/sqlparser/lib/src/analysis/types/resolving_visitor.dart +++ b/sqlparser/lib/src/analysis/types/resolving_visitor.dart @@ -46,7 +46,7 @@ class TypeResolver extends RecursiveVisitor { currentColumnIndex++; } else if (child is StarResultColumn) { - currentColumnIndex += child.scope.availableColumns.length; + currentColumnIndex += child.scope.expansionOfStarColumn?.length ?? 1; } else if (child is NestedQueryColumn) { visit(child.select, arg); } @@ -415,6 +415,19 @@ class TypeResolver extends RecursiveVisitor { visit(e.operand, const NoTypeExpectation()); } + @override + void visitStarFunctionParameter( + StarFunctionParameter e, TypeExpectation arg) { + final available = e.scope.expansionOfStarColumn; + if (available != null) { + // Make sure we resolve these columns, the type of some function + // invocation could depend on it. + for (final column in available) { + _handleColumn(column, e); + } + } + } + @override void visitStringComparison( StringComparisonExpression e, TypeExpectation arg) { diff --git a/sqlparser/lib/src/analysis/utils/expand_function_parameters.dart b/sqlparser/lib/src/analysis/utils/expand_function_parameters.dart index 58573de6..b788f951 100644 --- a/sqlparser/lib/src/analysis/utils/expand_function_parameters.dart +++ b/sqlparser/lib/src/analysis/utils/expand_function_parameters.dart @@ -6,7 +6,7 @@ extension ExpandParameters on SqlInvocation { /// Returns the expanded parameters of a function call. /// /// When a [StarFunctionParameter] is used, it's expanded to the - /// [ReferenceScope.availableColumns]. + /// [ReferenceScope.expansionOfStarColumn]. /// Elements of the result are either an [Expression] or a [Column]. List expandParameters() { final sqlParameters = parameters; @@ -16,7 +16,8 @@ extension ExpandParameters on SqlInvocation { } else if (sqlParameters is StarFunctionParameter) { // if * is used as a parameter, it refers to all columns in all tables // that are available in the current scope. - final allColumns = scope.availableColumns; + final allColumns = scope.expansionOfStarColumn; + if (allColumns == null) return const []; // When we look at `SELECT SUM(*), foo FROM ...`, the star in `SUM` // shouldn't expand to include itself. diff --git a/sqlparser/lib/src/ast/node.dart b/sqlparser/lib/src/ast/node.dart index cbee8710..4035fee6 100644 --- a/sqlparser/lib/src/ast/node.dart +++ b/sqlparser/lib/src/ast/node.dart @@ -93,9 +93,7 @@ abstract class AstNode with HasMetaMixin implements SyntacticEntity { return null; } - /// The [ReferenceScope], which contains available tables, column references - /// and functions for this node. - ReferenceScope get scope { + ReferenceScope? get optionalScope { AstNode? node = this; while (node != null) { @@ -104,9 +102,20 @@ abstract class AstNode with HasMetaMixin implements SyntacticEntity { node = node.parent; } + return null; + } + + /// The [ReferenceScope], which contains available tables, column references + /// and functions for this node. + ReferenceScope get scope { + final resolved = optionalScope; + if (resolved != null) return resolved; + throw StateError('No reference scope found in this or any parent node'); } + StatementScope get statementScope => StatementScope.cast(scope); + /// Applies a [ReferenceScope] to this node. Variables declared in [scope] /// will be visible to this node and to [allDescendants]. set scope(ReferenceScope scope) { diff --git a/sqlparser/lib/src/engine/sql_engine.dart b/sqlparser/lib/src/engine/sql_engine.dart index 88d6c641..e7407391 100644 --- a/sqlparser/lib/src/engine/sql_engine.dart +++ b/sqlparser/lib/src/engine/sql_engine.dart @@ -75,14 +75,15 @@ class SqlEngine { options.addTableValuedFunctionHandler(handler); } - ReferenceScope _constructRootScope({ReferenceScope? parent}) { - final scope = parent == null ? ReferenceScope(null) : parent.createChild(); + RootScope _constructRootScope() { + final scope = RootScope(); + for (final resultSet in knownResultSets) { - scope.register(resultSet.name, resultSet); + scope.knownTables[resultSet.name] = resultSet; } for (final module in _knownModules) { - scope.register(module.name, module); + scope.knownModules[module.name] = module; } return scope; @@ -127,7 +128,7 @@ class SqlEngine { Parser(tokensForParser, useDrift: true, autoComplete: autoComplete); final driftFile = parser.driftFile(); - _attachRootScope(driftFile); + driftFile.scope = _constructRootScope(); return ParseResult._( driftFile, tokens, parser.errors, content, autoComplete); @@ -190,13 +191,13 @@ class SqlEngine { AnalysisContext _createContext( AstNode node, String sql, AnalyzeStatementOptions? stmtOptions) { - return AnalysisContext(node, sql, options, + return AnalysisContext(node, sql, _constructRootScope(), options, stmtOptions: stmtOptions, schemaSupport: schemaReader); } void _analyzeContext(AnalysisContext context) { final node = context.root; - _attachRootScope(node); + node.scope = context.rootScope; try { AstPreparingVisitor(context: context).start(node); @@ -215,16 +216,6 @@ class SqlEngine { rethrow; } } - - void _attachRootScope(AstNode root) { - // calling node.referenceScope throws when no scope is set, we use the - // nullable variant here - final safeScope = root.selfAndParents - .map((node) => node.meta()) - .firstWhere((e) => e != null, orElse: () => null); - - root.scope = _constructRootScope(parent: safeScope); - } } /// The result of parsing an sql query. Contains the root of the AST and all diff --git a/sqlparser/lib/utils/case_insensitive_map.dart b/sqlparser/lib/utils/case_insensitive_map.dart new file mode 100644 index 00000000..3f1566df --- /dev/null +++ b/sqlparser/lib/utils/case_insensitive_map.dart @@ -0,0 +1,36 @@ +import 'dart:collection'; + +/// A map from strings to [T] where keys are compared without case sensitivity. +class CaseInsensitiveMap extends MapBase { + final Map _normalized = {}; + + @override + T? operator [](Object? key) { + if (key is String?) { + return _normalized[key?.toLowerCase()]; + } else { + return null; + } + } + + @override + void operator []=(K key, T value) { + _normalized[key?.toLowerCase() as K] = value; + } + + @override + void clear() { + _normalized.clear(); + } + + @override + Iterable get keys => _normalized.keys; + + @override + T? remove(Object? key) { + if (key is String?) { + return _normalized.remove(key?.toLowerCase()); + } + return null; + } +} diff --git a/sqlparser/test/analysis/available_tables_test.dart b/sqlparser/test/analysis/available_tables_test.dart index 676759d8..ff83c008 100644 --- a/sqlparser/test/analysis/available_tables_test.dart +++ b/sqlparser/test/analysis/available_tables_test.dart @@ -43,14 +43,14 @@ void main() { ); '''); - final result = engine.analyze(r''' + final result = engine.analyze(r''' SELECT d.*, c.** FROM with_defaults d LEFT OUTER JOIN with_constraints c ON d.a = c.a AND d.b = c.b WHERE $predicate; '''); - final scope = result.root.scope; - expect(scope.allOf(), hasLength(2)); + final scope = result.root.statementScope; + expect(scope.resultSets, hasLength(2)); }); } diff --git a/sqlparser/test/analysis/column_resolver_test.dart b/sqlparser/test/analysis/column_resolver_test.dart index 0797eefc..e0a25f57 100644 --- a/sqlparser/test/analysis/column_resolver_test.dart +++ b/sqlparser/test/analysis/column_resolver_test.dart @@ -2,6 +2,7 @@ import 'package:sqlparser/sqlparser.dart'; import 'package:test/test.dart'; import 'data.dart'; +import 'errors/utils.dart'; void main() { late SqlEngine engine; @@ -156,7 +157,7 @@ INSERT INTO demo VALUES (?, ?) final result = engine.analyze(''' UPDATE inventory SET quantity = quantity - daily.amt - FROM (SELECT sum(quantity) AS amt, itemId FROM sales GROUP BY 2) + FROM (SELECT sum(quantity) AS amt, itemId FROM sales GROUP BY 2) AS daily WHERE inventory.itemId = daily.itemId; '''); @@ -238,4 +239,18 @@ INSERT INTO demo VALUES (?, ?) expect(root.resolvedTargetColumns, hasLength(1)); }); }); + + test('does not allow a subquery in from to read outer values', () { + final result = engine.analyze( + 'SELECT * FROM demo d1, (SELECT * FROM demo i WHERE i.id = d1.id) d2;'); + + result.expectError('d1.id', type: AnalysisErrorType.referencedUnknownTable); + }); + + test('allows subquery expressions to read outer values', () { + final result = engine.analyze('SELECT * FROM demo d1 WHERE ' + 'EXISTS (SELECT * FROM demo i WHERE i.id = d1.id);'); + + result.expectNoError(); + }); } diff --git a/sqlparser/test/analysis/reference_resolver_test.dart b/sqlparser/test/analysis/reference_resolver_test.dart index 95e6562a..db2bee62 100644 --- a/sqlparser/test/analysis/reference_resolver_test.dart +++ b/sqlparser/test/analysis/reference_resolver_test.dart @@ -4,6 +4,7 @@ import 'package:test/test.dart'; import '../parser/utils.dart'; import 'data.dart'; +import 'errors/utils.dart'; void main() { test('correctly resolves return columns', () { @@ -94,6 +95,15 @@ void main() { ); }); + test('does not allow references to result column outside of ORDER BY', () { + final engine = SqlEngine()..registerTable(demoTable); + + final context = engine + .analyze('SELECT d.content, 3 * d.id AS t, t AS u FROM demo AS d'); + + context.expectError('t', type: AnalysisErrorType.referencedUnknownColumn); + }); + test('resolves columns from nested results', () { final engine = SqlEngine(EngineOptions(useDriftExtensions: true)) ..registerTable(demoTable) @@ -315,4 +325,39 @@ SELECT row_number() OVER wnd FROM demo testWith('DELETE FROM users RETURNING id;'); }); }); + + test('resolves column in foreign key declaration', () { + final engine = SqlEngine()..registerTableFromSql(''' +CREATE TABLE points ( + id INTEGER NOT NULL PRIMARY KEY, + lat REAL NOT NULL, + long REAL NOT NULL +); +'''); + + final parseResult = engine.parse(''' +CREATE TABLE routes ( + id INTEGER NOT NULL PRIMARY KEY, + "from" INTEGER NOT NULL REFERENCES points (id), + "to" INTEGER NOT NULL REFERENCES points (id) +); +'''); + final table = const SchemaFromCreateTable() + .read(parseResult.rootNode as CreateTableStatement); + engine.registerTable(table); + + final result = engine.analyzeParsed(parseResult); + + result.expectNoError(); + + final createTable = result.root as CreateTableStatement; + final fromReference = + createTable.columns[1].constraints[1] as ForeignKeyColumnConstraint; + final fromReferenced = + fromReference.clause.columnNames.single.resolvedColumn; + + expect(fromReferenced, isNotNull); + expect( + fromReferenced!.containingSet, result.rootScope.knownTables['points']); + }); } diff --git a/sqlparser/test/analysis/regression_test.dart b/sqlparser/test/analysis/regression_test.dart index 983b4c9a..e0ac01ae 100644 --- a/sqlparser/test/analysis/regression_test.dart +++ b/sqlparser/test/analysis/regression_test.dart @@ -53,19 +53,19 @@ void main() { final result = engine.analyze(''' WITH RECURSIVE employeeHierarchy(id, name, manager_id) AS ( - SELECT id, + SELECT id, name, manager_id FROM employees WHERE manager_id IS NULL UNION ALL - SELECT e.id, + SELECT e.id, e.name, e.manager_id FROM employees e JOIN employeeHierarchy ON e.manager_id = employeeHierarchy.id ) - SELECT e.id, + SELECT e.id, e.name, e.manager_id, n.note @@ -156,4 +156,36 @@ CREATE TABLE downloads ( .having((e) => e.type, 'type', BasicType.int) .having((e) => e.nullable, 'nullable', isFalse)); }); + + test('regression test for #1858', () { + // https://github.com/simolus3/drift/issues/1858 + final engine = SqlEngine( + EngineOptions(useDriftExtensions: true, version: SqliteVersion.v3_38)); + + engine.registerTableFromSql(''' +CREATE TABLE IF NOT EXISTS contract_has_add_fees +( + position_id INTEGER NOT NULL, + position_db INTEGER NOT NULL, + addfee TEXT NOT NULL, + amount REAL NOT NULL, + changed DATETIME, + removed BOOLEAN NOT NULL DEFAULT FALSE, + PRIMARY KEY (position_id, position_db, addfee) +) WITHOUT ROWID; +'''); + + final result = engine.analyze(r''' +DELETE +FROM contract_has_add_fees +WHERE EXISTS(SELECT * + FROM json_each(:json) AS j + WHERE position_id = json_extract(j.value, '$.positionid') + AND position_db = json_extract(j.value, '$.positiondb') + AND addfee = json_extract(j.value, '$.addfee') + ); +'''); + + expect(result.errors, isEmpty); + }); } diff --git a/sqlparser/test/analysis/types2/join_analysis_test.dart b/sqlparser/test/analysis/types2/join_analysis_test.dart index eb064f7b..87959216 100644 --- a/sqlparser/test/analysis/types2/join_analysis_test.dart +++ b/sqlparser/test/analysis/types2/join_analysis_test.dart @@ -18,24 +18,18 @@ void main() { final model = JoinModel.of(stmt)!; expect( - model.isNullableTable(stmt.scope - .resolve('a1')! - .resultSet - .resultSet!), + model.isNullableTable( + stmt.scope.resolveResultSet('a1')!.resultSet.resultSet!), isFalse, ); expect( - model.isNullableTable(stmt.scope - .resolve('a2')! - .resultSet - .resultSet!), + model.isNullableTable( + stmt.scope.resolveResultSet('a2')!.resultSet.resultSet!), isTrue, ); expect( - model.isNullableTable(stmt.scope - .resolve('a3')! - .resultSet - .resultSet!), + model.isNullableTable( + stmt.scope.resolveResultSet('a3')!.resultSet.resultSet!), isFalse, ); }); diff --git a/sqlparser/test/utils/find_referenced_tables_test.dart b/sqlparser/test/utils/find_referenced_tables_test.dart index d684aac7..9aec262e 100644 --- a/sqlparser/test/utils/find_referenced_tables_test.dart +++ b/sqlparser/test/utils/find_referenced_tables_test.dart @@ -67,6 +67,7 @@ void main() { INSERT INTO logins (user, timestamp) VALUES (new.id, 0); END; '''); + expect(ctx.errors, isEmpty); final body = (ctx.root as CreateTriggerStatement).action; // Users referenced via "new" in body.