Merge pull request #1859 from simolus3/new-column-resolver

New implementation for reference scopes

drift_dev/lib/src/analyzer/moor/create_table_reader.dart

@@ -34,7 +34,8 @@ class CreateTableReader {
     Table table;
     try {
       table = _schemaReader.read(stmt);
-    } catch (e) {
+    } catch (e, s) {
+      print(s);
       step.reportError(ErrorInMoorFile(
         span: stmt.tableNameToken!.span,
         message: 'Could not extract schema information for this table: $e',

drift_dev/lib/src/analyzer/sql_queries/type_mapping.dart

@@ -304,8 +304,7 @@ class TypeMapper {
       },
     );
 
-    final availableResults =
+    final availableResults = placeholder.statementScope.allAvailableResultSets;
-        placeholder.scope.allOf<ResultSetAvailableInStatement>();
     final availableMoorResults = <AvailableMoorResultSet>[];
     for (final available in availableResults) {
       final aliasedResultSet = available.resultSet.resultSet;

sqlparser/CHANGELOG.md

@@ -1,5 +1,6 @@
-## 0.21.1-dev
+## 0.22.0-dev
 
+- Refactor how tables and columns are resolved internally.
 - Lint for `DISTINCT` misuse in aggregate function calls.
 
 ## 0.21.0

sqlparser/lib/src/analysis/analysis.dart

@@ -7,6 +7,8 @@ import 'package:source_span/source_span.dart';
 import 'package:sqlparser/sqlparser.dart' hide ExpandParameters;
 import 'package:sqlparser/src/utils/meta.dart';
 
+import '../../utils/case_insensitive_map.dart';
+
 export 'types/data.dart';
 export 'types/types.dart' show TypeInferenceResults;
 

sqlparser/lib/src/analysis/context.dart

@@ -13,6 +13,11 @@ class AnalysisContext {
   /// The raw sql statement that was used to construct this [AnalysisContext].
   final String sql;
 
+  /// The root scope used when analyzing SQL statements.
+  ///
+  /// This contains information about known tables and modules.
+  final RootScope rootScope;
+
   /// Additional information about variables in this context, passed from the
   /// outside.
   final AnalyzeStatementOptions stmtOptions;
@@ -34,7 +39,7 @@ class AnalysisContext {
   late final TypeInferenceResults types2;
 
   /// Constructs a new analysis context from the AST and the source sql.
-  AnalysisContext(this.root, this.sql, this.engineOptions,
+  AnalysisContext(this.root, this.sql, this.rootScope, this.engineOptions,
       {AnalyzeStatementOptions? stmtOptions, required this.schemaSupport})
       : stmtOptions = stmtOptions ?? const AnalyzeStatementOptions();
 

sqlparser/lib/src/analysis/schema/from_create_table.dart

@@ -16,7 +16,7 @@ class SchemaFromCreateTable {
     if (stmt is CreateTableStatement) {
       return _readCreateTable(stmt);
     } else if (stmt is CreateVirtualTableStatement) {
-      final module = stmt.scope.resolve<Module>(stmt.moduleName);
+      final module = stmt.scope.rootScope.knownModules[stmt.moduleName];
 
       if (module == null) {
         throw CantReadSchemaException('Unknown module "${stmt.moduleName}", '

sqlparser/lib/src/analysis/schema/references.dart

@@ -20,126 +20,337 @@ mixin Referencable {
   bool get visibleToChildren => false;
 }
 
-/// Class which keeps track of references for tables, columns and functions in a
+/// A class managing which tables and columns are visible to which AST nodes.
-/// query.
+abstract class ReferenceScope {
-class ReferenceScope {
+  RootScope get rootScope;
-  final ReferenceScope? parent;
-  final ReferenceScope? root;
 
-  /// Whether the [availableColumns] of the [parent] are available in this scope
+  /// The list of column to which a `*` would expand to.
-  /// as well.
+  ///
-  final bool inheritAvailableColumns;
+  /// This is not necessary the same list of columns that could be resolved
+  /// through [resolveUnqualifiedReference]. For subquery expressions, columns
+  /// in parent scopes may be referenced without a qualified, but they don't
+  /// appear in a `*` expansion for the subquery.
+  List<Column>? get expansionOfStarColumn => null;
 
-  /// Gets the effective root scope. If no [root] scope has been set, this
+  /// Attempts to find a result set that has been added to this scope, for
-  /// scope is assumed to be the root scope.
+  /// instance because it was introduced in a `FROM` clause.
-  ReferenceScope get effectiveRoot => root ?? this;
+  ///
+  /// This is useful to resolve qualified references (e.g. to resolve `foo.bar`
+  /// the resolver would call [resolveResultSet]("foo") and then look up the
+  /// `bar` column in that result set).
+  ResultSetAvailableInStatement? resolveResultSet(String name) => null;
 
-  final Map<String, List<Referencable>> _references = {};
+  /// Adds an added result set to this scope.
+  ///
+  /// This operation is not supported for all kinds of scopes, a [StateError]
+  /// is thrown for invalid scopes.
+  void addResolvedResultSet(
+      String? name, ResultSetAvailableInStatement resultSet) {
+    throw StateError('Result set cannot be added in this scope: $this');
+  }
 
-  List<Column>? _availableColumns;
+  /// Registers a [ResultSetAvailableInStatement] to a [TableAlias] for the
+  /// given [resultSet].
+  ///
+  /// Like [addResolvedResultSet], this operation is not supported on all
+  /// scopes.
+  void addAlias(AstNode origin, ResultSet resultSet, String alias) {
+    final createdAlias = TableAlias(resultSet, alias);
+    addResolvedResultSet(
+        alias, ResultSetAvailableInStatement(origin, createdAlias));
+  }
 
-  /// All columns that would be available in this scope. Can be used to resolve
+  /// Attempts to find a result set that _can_ be added to a scope.
-  /// a '*' expression for function calls or result columns
+  ///
-  List<Column> get availableColumns {
+  /// This is used to resolve table references. Usually, after a result set to
-    if (_availableColumns != null) return _availableColumns!;
+  /// add has been resolve,d a [ResultSetAvailableInStatement] is added to the
-    if (inheritAvailableColumns) return parent!.availableColumns;
+  /// scope and [resolveResultSet] will find that afterwards.
+  ResultSet? resolveResultSetToAdd(String name) => rootScope.knownTables[name];
+
+  /// Attempts to resolve an unqualified reference from a [columnName].
+  ///
+  /// In sqlite, an `ORDER BY` column may refer to aliases of result columns
+  /// in the current statement: `SELECT foo AS bar FROM tbl ORDER BY bar` is
+  /// legal, but `SELECT foo AS bar FROM tbl WHERE bar < 10` is not. To control
+  /// whether result columns may be resolved, the [allowReferenceToResultColumn]
+  /// flag can be enabled.
+  ///
+  /// If an empty list is returned, the reference couldn't be resolved. If the
+  /// returned list contains more than one column, the lookup is ambigious.
+  List<Column> resolveUnqualifiedReference(String columnName,
+          {bool allowReferenceToResultColumn = false}) =>
+      const [];
+}
+
+/// The root scope created by the SQL engine to analyze a statement.
+///
+/// This contains known tables (or views) and modules to look up.
+class RootScope extends ReferenceScope {
+  @override
+  RootScope get rootScope => this;
+
+  /// All tables (or views, or other result sets) that are known in the current
+  /// schema.
+  ///
+  /// [resolveResultSetToAdd] will query these tables by default.
+  final Map<String, ResultSet> knownTables = CaseInsensitiveMap();
+
+  /// Known modules that are registered for this statement.
+  ///
+  /// This is used to resolve `CREATE VIRTUAL TABLE` statements.
+  final Map<String, Module> knownModules = CaseInsensitiveMap();
+}
+
+/// A scope used by statements.
+///
+/// Tables added from `FROM` clauses are added to [resultSets], CTEs are added
+/// to [additionalKnownTables].
+///
+/// This is the scope most commonly used, but specific nodes may be attached to
+/// a different scope in case they have limited visibility. For instance,
+///  - foreign key clauses are wrapped in a [SingleTableReferenceScope] because
+///    they can't see unqualified columns of the overal scope.
+///  - subquery expressions can see parent tables and columns, but their columns
+///    aren't visible in the parent statement. This is implemented by wrapping
+///    them in a [StatementScope] as well.
+///  - subqueries appearing in a `FROM` clause _can't_ see outer columns and
+///    tables. These statements are also wrapped in a [StatementScope], but a
+///    [SubqueryInFromScope] is insertted as an intermediatet scope to prevent
+///    the inner scope from seeing the outer columns.
+
+class StatementScope extends ReferenceScope {
+  final ReferenceScope parent;
+
+  /// Additional tables (that haven't necessarily been added in a `FROM` clause
+  /// that are only visible in this scope).
+  ///
+  /// This is commonly used for common table expressions, e.g a `WITH foo AS
+  /// (...)` would add a result set `foo` into the [additionalKnownTables] of
+  /// the overall statement, because `foo` can now be selected.
+  final Map<String, ResultSet> additionalKnownTables = CaseInsensitiveMap();
+
+  /// Result sets that were added through a `FROM` clause and are now available
+  /// in this scope.
+  ///
+  /// The [ResultSetAvailableInStatement] contains information about the AST
+  /// node causing this statement to be available.
+  final Map<String?, ResultSetAvailableInStatement> resultSets =
+      CaseInsensitiveMap();
+
+  /// For select statements, additional columns available under a name because
+  /// there were added after the `SELECT`.
+  ///
+  /// This is used to resolve unqualified references by `ORDER BY` clauses.
+  final List<Column> namedResultColumns = [];
+
+  final Map<String, NamedWindowDeclaration> windowDeclarations =
+      CaseInsensitiveMap();
+
+  /// All columns that a (unqualified) `*` in a select statement or function
+  /// call argument would expand to.
+  @override
+  List<Column>? expansionOfStarColumn;
+
+  StatementScope(this.parent);
+
+  StatementScope? get parentStatementScope {
+    final parent = this.parent;
+    if (parent is StatementScope) {
+      return parent;
+    } else if (parent is MiscStatementSubScope) {
+      return parent.parent;
+    } else {
+      return null;
+    }
+  }
+
+  /// All result sets available in this and parent scopes.
+  Iterable<ResultSetAvailableInStatement> get allAvailableResultSets {
+    final here = resultSets.values;
+    final parent = parentStatementScope;
+    return parent != null
+        ? here.followedBy(parent.allAvailableResultSets)
+        : here;
+  }
+
+  @override
+  RootScope get rootScope => parent.rootScope;
+
+  @override
+  void addAlias(AstNode origin, ResultSet resultSet, String alias) {
+    final createdAlias = TableAlias(resultSet, alias);
+    additionalKnownTables[alias] = createdAlias;
+    resultSets[alias] = ResultSetAvailableInStatement(origin, createdAlias);
+  }
+
+  @override
+  ResultSetAvailableInStatement? resolveResultSet(String name) {
+    return resultSets[name] ?? parentStatementScope?.resolveResultSet(name);
+  }
+
+  @override
+  void addResolvedResultSet(
+      String? name, ResultSetAvailableInStatement resultSet) {
+    resultSets[name] = resultSet;
+  }
+
+  @override
+  ResultSet? resolveResultSetToAdd(String name) {
+    return additionalKnownTables[name] ??
+        parentStatementScope?.resolveResultSetToAdd(name) ??
+        rootScope.knownTables[name];
+  }
+
+  @override
+  List<Column> resolveUnqualifiedReference(String columnName,
+      {bool allowReferenceToResultColumn = false}) {
+    if (allowReferenceToResultColumn) {
+      final foundColumn = namedResultColumns.firstWhereOrNull(
+          (c) => c.name.toLowerCase() == columnName.toLowerCase());
+      if (foundColumn != null) {
+        return [foundColumn];
+      }
+    }
+
+    StatementScope? currentScope = this;
+
+    // Search scopes for a matching column in an added result set. If a column
+    // reference is found in a closer scope, it takes precedence over outer
+    // scopes. However, it's an error if two columns with the same name are
+    // found in the same scope.
+    while (currentScope != null) {
+      final available = currentScope.resultSets.values;
+      final sourceColumns = <Column>{};
+      final availableColumns = <AvailableColumn>[];
+
+      for (final availableSource in available) {
+        final resolvedColumns =
+            availableSource.resultSet.resultSet?.resolvedColumns;
+        if (resolvedColumns == null) continue;
+
+        for (final column in resolvedColumns) {
+          if (column.name.toLowerCase() == columnName.toLowerCase() &&
+              sourceColumns.add(column)) {
+            availableColumns.add(AvailableColumn(column, availableSource));
+          }
+        }
+      }
+
+      if (availableColumns.isEmpty) {
+        currentScope = currentScope.parentStatementScope;
+        if (currentScope == null) {
+          // Reached the outermost scope without finding a reference target.
+          return const [];
+        }
+        continue;
+      } else {
+        return availableColumns;
+      }
+    }
 
     return const [];
   }
 
-  set availableColumns(List<Column>? value) {
+  factory StatementScope.forStatement(RootScope root, Statement statement) {
-    // guard against lists of subtype of column
+    return StatementScope(statement.optionalScope ?? root);
-    if (value != null) {
+  }
-      _availableColumns = <Column>[...value];
+
+  static StatementScope cast(ReferenceScope other) {
+    if (other is StatementScope) {
+      return other;
+    } else if (other is MiscStatementSubScope) {
+      return other.parent;
     } else {
-      _availableColumns = null;
+      throw ArgumentError.value(
+          other, 'other', 'Not resolvable to a statement scope');
+    }
+  }
+}
+
+/// A special intermediate scope used for subqueries appearing in a `FROM`
+/// clause so that the subquery can't see outer columns and tables being added.
+class SubqueryInFromScope extends ReferenceScope {
+  final StatementScope enclosingStatement;
+
+  SubqueryInFromScope(this.enclosingStatement);
+
+  @override
+  RootScope get rootScope => enclosingStatement.rootScope;
+}
+
+/// A rarely used sub-scope for AST nodes that belong to a statement, but may
+/// have access to more result sets.
+///
+/// For instance, the body of an `ON CONFLICT DO UPDATE`-clause may refer to a
+/// table alias `excluded` to get access to a conflicting table.
+class MiscStatementSubScope extends ReferenceScope {
+  final StatementScope parent;
+
+  final Map<String?, ResultSetAvailableInStatement> additionalResultSets =
+      CaseInsensitiveMap();
+
+  MiscStatementSubScope(this.parent);
+
+  @override
+  RootScope get rootScope => parent.rootScope;
+
+  @override
+  ResultSetAvailableInStatement? resolveResultSet(String name) {
+    return additionalResultSets[name] ?? parent.resolveResultSet(name);
+  }
+
+  @override
+  void addResolvedResultSet(
+      String? name, ResultSetAvailableInStatement resultSet) {
+    additionalResultSets[name] = resultSet;
+  }
+
+  @override
+  List<Column> resolveUnqualifiedReference(String columnName,
+      {bool allowReferenceToResultColumn = false}) {
+    return parent.resolveUnqualifiedReference(columnName);
+  }
+}
+
+/// A reference scope that only allows a single added result set.
+///
+/// This is used for e.g. foreign key clauses (`REFERENCES table (a, b, c)`),
+/// where `a`, `b` and `c` can only refer to `table`.
+class SingleTableReferenceScope extends ReferenceScope {
+  final ReferenceScope parent;
+
+  String? addedTableName;
+  ResultSetAvailableInStatement? addedTable;
+
+  SingleTableReferenceScope(this.parent);
+
+  @override
+  RootScope get rootScope => parent.rootScope;
+
+  @override
+  ResultSetAvailableInStatement? resolveResultSet(String name) {
+    if (name == addedTableName) {
+      return addedTable;
+    } else {
+      return null;
+    }
+  }
+
+  @override
+  void addResolvedResultSet(
+      String? name, ResultSetAvailableInStatement resultSet) {
+    addedTableName = null;
+    addedTable = null;
+  }
+
+  @override
+  List<Column> resolveUnqualifiedReference(String columnName,
+      {bool allowReferenceToResultColumn = false}) {
+    final column = addedTable?.resultSet.resultSet?.findColumn(columnName);
+    if (column != null) {
+      return [AvailableColumn(column, addedTable!)];
+    } else {
+      return const [];
     }
   }
-
-  ReferenceScope(this.parent,
-      {this.root, this.inheritAvailableColumns = false});
-
-  void addAvailableColumn(Column column) {
-    // make sure _availableColumns is resolved and mutable
-    final ownColumns = _availableColumns ??= <Column>[...availableColumns];
-    ownColumns.add(column);
-  }
-
-  ReferenceScope createChild({bool? inheritAvailableColumns}) {
-    // wonder why we're creating a linked list of reference scopes instead of
-    // just passing down a copy of [_references]? In sql, some variables can be
-    // used before they're defined, even in child scopes:
-    // SELECT *, (SELECT * FROM table2 WHERE id = t1.a) FROM table2 t1
-    return ReferenceScope(
-      this,
-      root: effectiveRoot,
-      inheritAvailableColumns: inheritAvailableColumns ?? false,
-    );
-  }
-
-  ReferenceScope createSibling() {
-    return parent!.createChild();
-  }
-
-  /// Registers something that can be referenced in this and child scopes.
-  void register(String identifier, Referencable ref) {
-    _references.putIfAbsent(identifier.toUpperCase(), () => []).add(ref);
-  }
-
-  /// Registers both a [TableAlias] and a [ResultSetAvailableInStatement] so
-  /// that the alias can be used in expressions without being selected.
-  void registerUsableAlias(AstNode origin, ResultSet resultSet, String alias) {
-    final createdAlias = TableAlias(resultSet, alias);
-    register(alias, createdAlias);
-    register(alias, ResultSetAvailableInStatement(origin, createdAlias));
-  }
-
-  /// Resolves to a [Referencable] with the given [name] and of the type [T].
-  /// If the reference couldn't be found, null is returned and [orElse] will be
-  /// called.
-  T? resolve<T extends Referencable>(String name, {Function()? orElse}) {
-    ReferenceScope? scope = this;
-    var isAtParent = false;
-    final upper = name.toUpperCase();
-
-    while (scope != null) {
-      if (scope._references.containsKey(upper)) {
-        final candidates = scope._references[upper]!;
-        final resolved = candidates.whereType<T>().where((x) {
-          return x.visibleToChildren || !isAtParent;
-        });
-        if (resolved.isNotEmpty) {
-          return resolved.first;
-        }
-      }
-
-      scope = scope.parent;
-      isAtParent = true;
-    }
-
-    if (orElse != null) orElse();
-    return null; // not found in any parent scope
-  }
-
-  /// Returns everything that is in scope and a subtype of [T].
-  List<T> allOf<T extends Referencable>() {
-    ReferenceScope? scope = this;
-    var isInCurrentScope = true;
-    final collected = <T>[];
-
-    while (scope != null) {
-      var foundValues =
-          scope._references.values.expand((list) => list).whereType<T>();
-
-      if (!isInCurrentScope) {
-        foundValues =
-            foundValues.where((element) => element.visibleToChildren).cast();
-      }
-
-      collected.addAll(foundValues);
-      scope = scope.parent;
-      isInCurrentScope = false;
-    }
-    return collected;
-  }
 }

sqlparser/lib/src/analysis/schema/table.dart

@@ -80,6 +80,9 @@ class Table extends NamedResultSet with HasMetaMixin implements HumanReadable {
   String humanReadableDescription() {
     return name;
   }
+
+  @override
+  String toString() => 'Table $name';
 }
 
 class TableAlias extends NamedResultSet implements HumanReadable {

sqlparser/lib/src/analysis/steps/column_resolver.dart

@@ -57,7 +57,7 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
     if (table != null) {
       // add "excluded" table qualifier that referring to the row that would
       // have been inserted had the uniqueness constraint not been violated.
-      e.scope.registerUsableAlias(e, table, 'excluded');
+      e.scope.addAlias(e, table, 'excluded');
     }
 
     visitChildren(e, arg);
@@ -80,7 +80,7 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
     final from = e.from;
     if (from != null) _handle(from, availableColumns);
 
-    e.scope.availableColumns = availableColumns;
+    e.statementScope.expansionOfStarColumn = availableColumns;
     for (final child in e.childNodes) {
       // Visit remaining children
       if (child != e.table && child != e.from) visit(child, arg);
@@ -92,7 +92,7 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
   ResultSet? _addIfResolved(AstNode node, TableReference ref) {
     final table = _resolveTableReference(ref);
     if (table != null) {
-      node.scope.availableColumns = table.resolvedColumns;
+      node.statementScope.expansionOfStarColumn = table.resolvedColumns;
     }
 
     return table;
@@ -135,7 +135,7 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
     final clause = stmt.returning;
     if (clause == null) return;
 
-    final columns = _resolveColumns(stmt.scope, clause.columns,
+    final columns = _resolveColumns(stmt.statementScope, clause.columns,
         columnsForStar: mainTable?.resolvedColumns);
     stmt.returnedResultSet = CustomResultSet(columns);
   }
@@ -149,16 +149,16 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
       return;
     }
 
-    final scope = e.scope;
+    final scope = e.statementScope;
 
     // Add columns of the target table for when and update of clauses
-    scope.availableColumns = table.resolvedColumns;
+    scope.expansionOfStarColumn = table.resolvedColumns;
 
     if (e.target.introducesNew) {
-      scope.registerUsableAlias(e, table, 'new');
+      scope.addAlias(e, table, 'new');
     }
     if (e.target.introducesOld) {
-      scope.registerUsableAlias(e, table, 'old');
+      scope.addAlias(e, table, 'old');
     }
 
     visitChildren(e, arg);
@@ -236,16 +236,16 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
       _handle(s.from!, availableColumns);
     }
 
-    final scope = s.scope;
+    final scope = s.statementScope;
-    scope.availableColumns = availableColumns;
+    scope.expansionOfStarColumn = availableColumns;
 
     s.resolvedColumns = _resolveColumns(scope, s.columns);
   }
 
-  List<Column> _resolveColumns(ReferenceScope scope, List<ResultColumn> columns,
+  List<Column> _resolveColumns(StatementScope scope, List<ResultColumn> columns,
       {List<Column>? columnsForStar}) {
     final usedColumns = <Column>[];
-    final availableColumns = <Column>[...scope.availableColumns];
+    final availableColumns = <Column>[...?scope.expansionOfStarColumn];
 
     // a select statement can include everything from its sub queries as a
     // result, but also expressions that appear as result columns
@@ -254,15 +254,15 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
         Iterable<Column>? visibleColumnsForStar;
 
         if (resultColumn.tableName != null) {
-          final tableResolver = scope.resolve<ResultSetAvailableInStatement>(
+          final tableResolver = scope.resolveResultSet(resultColumn.tableName!);
-              resultColumn.tableName!, orElse: () {
+          if (tableResolver == null) {
             context.reportError(AnalysisError(
               type: AnalysisErrorType.referencedUnknownTable,
               message: 'Unknown table: ${resultColumn.tableName}',
               relevantNode: resultColumn,
             ));
-          });
+            continue;
-          if (tableResolver == null) continue;
+          }
 
           visibleColumnsForStar =
               tableResolver.resultSet.resultSet?.resolvedColumns?.map(
@@ -274,7 +274,7 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
 
           // Star columns can't be used without a table (e.g. `SELECT *` is
           // not allowed)
-          if (scope.allOf<ResultSetAvailableInStatement>().isEmpty) {
+          if (scope.resultSets.isEmpty) {
             context.reportError(AnalysisError(
               type: AnalysisErrorType.starColumnWithoutTable,
               message: "Can't use * when no tables have been added",
@@ -306,20 +306,22 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
           final name = resultColumn.as;
           if (!availableColumns.any((c) => c.name == name)) {
             availableColumns.add(column);
-            scope.addAvailableColumn(column);
+            scope.namedResultColumns.add(column);
           }
         }
       } else if (resultColumn is NestedStarResultColumn) {
-        final target = scope.resolve<ResultSetAvailableInStatement>(
+        final target = scope.resolveResultSet(resultColumn.tableName);
-            resultColumn.tableName, orElse: () {
+
+        if (target == null) {
           context.reportError(AnalysisError(
             type: AnalysisErrorType.referencedUnknownTable,
             message: 'Unknown table: ${resultColumn.tableName}',
             relevantNode: resultColumn,
           ));
-        });
+          continue;
+        }
 
-        if (target != null) resultColumn.resultSet = target.resultSet.resultSet;
+        resultColumn.resultSet = target.resultSet.resultSet;
       } else if (resultColumn is NestedQueryColumn) {
         _resolveSelect(resultColumn.select);
       }
@@ -410,9 +412,8 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
 
     // Try resolving to a top-level table in the schema and to a result set that
     // may have been added to the table
-    final resolvedInSchema = scope.resolve<ResultSet>(r.tableName);
+    final resolvedInSchema = scope.resolveResultSetToAdd(r.tableName);
-    final resolvedInQuery =
+    final resolvedInQuery = scope.resolveResultSet(r.tableName);
-        scope.resolve<ResultSetAvailableInStatement>(r.tableName);
     final createdName = r.as;
 
     // Prefer using a table that has already been added if this isn't the
@@ -428,26 +429,27 @@ class ColumnResolver extends RecursiveVisitor<void, void> {
           ? TableAlias(resolvedInSchema, createdName)
           : resolvedInSchema;
     } else {
-      final available = scope
+      Iterable<String>? available;
-          .allOf<ResolvesToResultSet>()
-          .followedBy(scope
-              .allOf<ResultSetAvailableInStatement>()
-              .map((added) => added.resultSet))
-          .where((e) => e.resultSet != null)
-          .map((t) {
-        final resultSet = t.resultSet;
-        if (resultSet is HumanReadable) {
-          return (resultSet as HumanReadable).humanReadableDescription();
-        }
 
-        return t.toString();
+      if (scope is StatementScope) {
-      });
+        available = StatementScope.cast(scope)
+            .allAvailableResultSets
+            .where((e) => e.resultSet.resultSet != null)
+            .map((t) {
+          final resultSet = t.resultSet.resultSet;
+          if (resultSet is HumanReadable) {
+            return (resultSet as HumanReadable).humanReadableDescription();
+          }
+
+          return t.toString();
+        });
+      }
 
       context.reportError(UnresolvedReferenceError(
         type: AnalysisErrorType.referencedUnknownTable,
         relevantNode: r,
         reference: r.tableName,
-        available: available,
+        available: available ?? const Iterable.empty(),
       ));
     }
 

sqlparser/lib/src/analysis/steps/prepare_ast.dart

@@ -21,16 +21,15 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
 
   @override
   void visitCreateTableStatement(CreateTableStatement e, void arg) {
-    final scope = e.scope = e.scope.createChild();
+    final scope = e.scope = StatementScope.forStatement(context.rootScope, e);
-    final registeredTable = scope.resolve(e.tableName) as Table?;
+    final knownTable = context.rootScope.knownTables[e.tableName];
 
     // This is used so that tables can refer to their own columns. Code using
     // tables would first register the table and then run analysis again.
-    if (registeredTable != null) {
+    if (knownTable is Table) {
-      scope.availableColumns = registeredTable.resolvedColumns;
+      scope
-      for (final column in registeredTable.resolvedColumns) {
+        ..expansionOfStarColumn = knownTable.resolvedColumns
-        scope.register(column.name, column);
+        ..resultSets[null] = ResultSetAvailableInStatement(e, knownTable);
-      }
     }
 
     visitChildren(e, arg);
@@ -38,7 +37,7 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
 
   @override
   void visitCreateViewStatement(CreateViewStatement e, void arg) {
-    e.scope = e.scope.createChild();
+    e.scope = StatementScope.forStatement(context.rootScope, e);
     visitChildren(e, arg);
   }
 
@@ -54,20 +53,21 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
     //   (SELECT * FROM demo i WHERE i.id = d1.id) d2;"
     // it won't work.
     final isInFROM = e.parent is Queryable;
-    final scope = e.scope;
+    StatementScope scope;
 
     if (isInFROM) {
-      final surroundingSelect =
+      final surroundingSelect = e.parents
-          e.parents.firstWhere((node) => node is HasFrom).scope;
+          .firstWhere((node) => node is HasFrom)
-      final forked = surroundingSelect.createSibling();
+          .scope as StatementScope;
-      e.scope = forked;
+      scope = StatementScope(SubqueryInFromScope(surroundingSelect));
     } else {
-      final forked = scope.createChild();
+      scope = StatementScope.forStatement(context.rootScope, e);
-      e.scope = forked;
     }
 
+    e.scope = scope;
+
     for (final windowDecl in e.windowDeclarations) {
-      e.scope.register(windowDecl.name, windowDecl);
+      scope.windowDeclarations[windowDecl.name] = windowDecl;
     }
 
     // only the last statement in a compound select statement may have an order
@@ -107,18 +107,6 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
     visitChildren(e, arg);
   }
 
-  @override
-  void visitResultColumn(ResultColumn e, void arg) {
-    if (e is StarResultColumn) {
-      // doesn't need special treatment, star expressions can't be referenced
-    } else if (e is ExpressionResultColumn) {
-      if (e.as != null) {
-        e.scope.register(e.as!, e);
-      }
-    }
-    visitChildren(e, arg);
-  }
-
   @override
   void defaultQueryable(Queryable e, void arg) {
     final scope = e.scope;
@@ -128,14 +116,12 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
         final added = ResultSetAvailableInStatement(table, table);
         table.availableResultSet = added;
 
-        scope.register(table.as ?? table.tableName, added);
+        scope.addResolvedResultSet(table.as ?? table.tableName, added);
       },
       isSelect: (select) {
-        if (select.as != null) {
+        final added = ResultSetAvailableInStatement(select, select.statement);
-          final added = ResultSetAvailableInStatement(select, select.statement);
+        select.availableResultSet = added;
-          select.availableResultSet = added;
+        scope.addResolvedResultSet(select.as, added);
-          scope.register(select.as!, added);
-        }
       },
       isJoin: (join) {
         // the join can contain multiple tables. Luckily for us, all of them are
@@ -145,7 +131,7 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
       isTableFunction: (function) {
         final added = ResultSetAvailableInStatement(function, function);
         function.availableResultSet = added;
-        scope.register(function.as ?? function.name, added);
+        scope.addResolvedResultSet(function.as ?? function.name, added);
       },
     );
 
@@ -154,7 +140,13 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
 
   @override
   void visitCommonTableExpression(CommonTableExpression e, void arg) {
-    e.scope.register(e.cteTableName, e);
+    StatementScope.cast(e.scope).additionalKnownTables[e.cteTableName] = e;
+    visitChildren(e, arg);
+  }
+
+  @override
+  void visitForeignKeyClause(ForeignKeyClause e, void arg) {
+    e.scope = SingleTableReferenceScope(e.scope);
     visitChildren(e, arg);
   }
 
@@ -203,16 +195,11 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
     }
   }
 
-  void _forkScope(AstNode node, {bool? inheritAvailableColumns}) {
-    node.scope = node.scope
-        .createChild(inheritAvailableColumns: inheritAvailableColumns);
-  }
-
   @override
   void defaultNode(AstNode e, void arg) {
     // hack to fork scopes on statements (selects are handled above)
     if (e is Statement && e is! SelectStatement) {
-      _forkScope(e);
+      e.scope = StatementScope.forStatement(context.rootScope, e);
     }
 
     visitChildren(e, arg);
@@ -244,7 +231,7 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
     // "excluded" can be referred. Setting that row happens in the column
     // resolver
     if (e.action is DoUpdate) {
-      _forkScope(e.action, inheritAvailableColumns: true);
+      e.action.scope = MiscStatementSubScope(e.scope as StatementScope);
     }
 
     visitChildren(e, null);
@@ -257,7 +244,7 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
       // create a new scope for the nested query to differentiate between
       // references that can be resolved in the nested query and references
       // which require data from the parent query
-      e.select.scope = e.scope.createChild();
+      e.select.scope = MiscStatementSubScope(e.scope as StatementScope);
       AstPreparingVisitor(context: context).start(e.select);
     } else {
       super.visitDriftSpecificNode(e, arg);

sqlparser/lib/src/analysis/steps/reference_resolver.dart

@@ -43,8 +43,7 @@ class ReferenceResolver extends RecursiveVisitor<void, void> {
     if (e.entityName != null) {
       // first find the referenced table or view,
       // then use the column on that table or view.
-      final entityResolver =
+      final entityResolver = scope.resolveResultSet(e.entityName!);
-          scope.resolve<ResultSetAvailableInStatement>(e.entityName!);
       final resultSet = entityResolver?.resultSet.resultSet;
 
       if (resultSet == null) {
@@ -68,15 +67,19 @@ class ReferenceResolver extends RecursiveVisitor<void, void> {
     } else {
       // find any column with the referenced name.
       // todo special case for USING (...) in joins?
-      final columns =
+      final found = scope.resolveUnqualifiedReference(
-          scope.availableColumns.where((c) => c.name == e.columnName).toSet();
+        e.columnName,
+        // According to https://www.sqlite.org/lang_select.html#the_order_by_clause,
+        // a simple reference in an ordering term can refer to an output column.
+        allowReferenceToResultColumn: e.parent is OrderingTerm,
+      );
 
-      if (columns.isEmpty) {
+      if (found.isEmpty) {
         _reportUnknownColumnError(e);
       } else {
-        if (columns.length > 1) {
+        if (found.length > 1) {
           final description =
-              columns.map((c) => c.humanReadableDescription()).join(', ');
+              found.map((c) => c.humanReadableDescription()).join(', ');
 
           context.reportError(AnalysisError(
             type: AnalysisErrorType.ambiguousReference,
@@ -85,7 +88,7 @@ class ReferenceResolver extends RecursiveVisitor<void, void> {
           ));
         }
 
-        e.resolved = columns.first;
+        e.resolved = found.first;
       }
     }
 
@@ -108,23 +111,25 @@ class ReferenceResolver extends RecursiveVisitor<void, void> {
   @override
   void visitWindowFunctionInvocation(WindowFunctionInvocation e, void arg) {
     if (e.windowName != null && e.resolved == null) {
-      final resolved = e.scope.resolve<NamedWindowDeclaration>(e.windowName!);
+      e.resolved =
-      e.resolved = resolved;
+          StatementScope.cast(e.scope).windowDeclarations[e.windowName!];
     }
 
     visitChildren(e, arg);
   }
 
   void _reportUnknownColumnError(Reference e, {Iterable<Column>? columns}) {
-    columns ??= e.scope.availableColumns;
+    final msg = StringBuffer('Unknown column.');
-    final columnNames = e.scope.availableColumns
+    if (columns != null) {
-        .map((c) => c.humanReadableDescription())
+      final columnNames =
-        .join(', ');
+          columns.map((c) => c.humanReadableDescription()).join(', ');
+      msg.write(' Thesecolumns are available: $columnNames');
+    }
 
     context.reportError(AnalysisError(
       type: AnalysisErrorType.referencedUnknownColumn,
       relevantNode: e,
-      message: 'Unknown column. These columns are available: $columnNames',
+      message: msg.toString(),
     ));
   }
 

sqlparser/lib/src/analysis/types/resolving_visitor.dart

@@ -46,7 +46,7 @@ class TypeResolver extends RecursiveVisitor<TypeExpectation, void> {
 
           currentColumnIndex++;
         } else if (child is StarResultColumn) {
-          currentColumnIndex += child.scope.availableColumns.length;
+          currentColumnIndex += child.scope.expansionOfStarColumn?.length ?? 1;
         } else if (child is NestedQueryColumn) {
           visit(child.select, arg);
         }
@@ -415,6 +415,19 @@ class TypeResolver extends RecursiveVisitor<TypeExpectation, void> {
     visit(e.operand, const NoTypeExpectation());
   }
 
+  @override
+  void visitStarFunctionParameter(
+      StarFunctionParameter e, TypeExpectation arg) {
+    final available = e.scope.expansionOfStarColumn;
+    if (available != null) {
+      // Make sure we resolve these columns, the type of some function
+      // invocation could depend on it.
+      for (final column in available) {
+        _handleColumn(column, e);
+      }
+    }
+  }
+
   @override
   void visitStringComparison(
       StringComparisonExpression e, TypeExpectation arg) {

sqlparser/lib/src/analysis/utils/expand_function_parameters.dart

@@ -6,7 +6,7 @@ extension ExpandParameters on SqlInvocation {
   /// Returns the expanded parameters of a function call.
   ///
   /// When a [StarFunctionParameter] is used, it's expanded to the
-  /// [ReferenceScope.availableColumns].
+  /// [ReferenceScope.expansionOfStarColumn].
   /// Elements of the result are either an [Expression] or a [Column].
   List<Typeable> expandParameters() {
     final sqlParameters = parameters;
@@ -16,7 +16,8 @@ extension ExpandParameters on SqlInvocation {
     } else if (sqlParameters is StarFunctionParameter) {
       // if * is used as a parameter, it refers to all columns in all tables
       // that are available in the current scope.
-      final allColumns = scope.availableColumns;
+      final allColumns = scope.expansionOfStarColumn;
+      if (allColumns == null) return const [];
 
       // When we look at `SELECT SUM(*), foo FROM ...`, the star in `SUM`
       // shouldn't expand to include itself.

sqlparser/lib/src/ast/node.dart

@@ -93,9 +93,7 @@ abstract class AstNode with HasMetaMixin implements SyntacticEntity {
     return null;
   }
 
-  /// The [ReferenceScope], which contains available tables, column references
+  ReferenceScope? get optionalScope {
-  /// and functions for this node.
-  ReferenceScope get scope {
     AstNode? node = this;
 
     while (node != null) {
@@ -104,9 +102,20 @@ abstract class AstNode with HasMetaMixin implements SyntacticEntity {
       node = node.parent;
     }
 
+    return null;
+  }
+
+  /// The [ReferenceScope], which contains available tables, column references
+  /// and functions for this node.
+  ReferenceScope get scope {
+    final resolved = optionalScope;
+    if (resolved != null) return resolved;
+
     throw StateError('No reference scope found in this or any parent node');
   }
 
+  StatementScope get statementScope => StatementScope.cast(scope);
+
   /// Applies a [ReferenceScope] to this node. Variables declared in [scope]
   /// will be visible to this node and to [allDescendants].
   set scope(ReferenceScope scope) {

sqlparser/lib/src/engine/sql_engine.dart

@@ -75,14 +75,15 @@ class SqlEngine {
     options.addTableValuedFunctionHandler(handler);
   }
 
-  ReferenceScope _constructRootScope({ReferenceScope? parent}) {
+  RootScope _constructRootScope() {
-    final scope = parent == null ? ReferenceScope(null) : parent.createChild();
+    final scope = RootScope();
+
     for (final resultSet in knownResultSets) {
-      scope.register(resultSet.name, resultSet);
+      scope.knownTables[resultSet.name] = resultSet;
     }
 
     for (final module in _knownModules) {
-      scope.register(module.name, module);
+      scope.knownModules[module.name] = module;
     }
 
     return scope;
@@ -127,7 +128,7 @@ class SqlEngine {
         Parser(tokensForParser, useDrift: true, autoComplete: autoComplete);
 
     final driftFile = parser.driftFile();
-    _attachRootScope(driftFile);
+    driftFile.scope = _constructRootScope();
 
     return ParseResult._(
         driftFile, tokens, parser.errors, content, autoComplete);
@@ -190,13 +191,13 @@ class SqlEngine {
 
   AnalysisContext _createContext(
       AstNode node, String sql, AnalyzeStatementOptions? stmtOptions) {
-    return AnalysisContext(node, sql, options,
+    return AnalysisContext(node, sql, _constructRootScope(), options,
         stmtOptions: stmtOptions, schemaSupport: schemaReader);
   }
 
   void _analyzeContext(AnalysisContext context) {
     final node = context.root;
-    _attachRootScope(node);
+    node.scope = context.rootScope;
 
     try {
       AstPreparingVisitor(context: context).start(node);
@@ -215,16 +216,6 @@ class SqlEngine {
       rethrow;
     }
   }
-
-  void _attachRootScope(AstNode root) {
-    // calling node.referenceScope throws when no scope is set, we use the
-    // nullable variant here
-    final safeScope = root.selfAndParents
-        .map((node) => node.meta<ReferenceScope>())
-        .firstWhere((e) => e != null, orElse: () => null);
-
-    root.scope = _constructRootScope(parent: safeScope);
-  }
 }
 
 /// The result of parsing an sql query. Contains the root of the AST and all

sqlparser/lib/utils/case_insensitive_map.dart

@@ -0,0 +1,36 @@
+import 'dart:collection';
+
+/// A map from strings to [T] where keys are compared without case sensitivity.
+class CaseInsensitiveMap<K extends String?, T> extends MapBase<K, T> {
+  final Map<K, T> _normalized = {};
+
+  @override
+  T? operator [](Object? key) {
+    if (key is String?) {
+      return _normalized[key?.toLowerCase()];
+    } else {
+      return null;
+    }
+  }
+
+  @override
+  void operator []=(K key, T value) {
+    _normalized[key?.toLowerCase() as K] = value;
+  }
+
+  @override
+  void clear() {
+    _normalized.clear();
+  }
+
+  @override
+  Iterable<K> get keys => _normalized.keys;
+
+  @override
+  T? remove(Object? key) {
+    if (key is String?) {
+      return _normalized.remove(key?.toLowerCase());
+    }
+    return null;
+  }
+}

sqlparser/test/analysis/available_tables_test.dart

@@ -50,7 +50,7 @@ void main() {
       WHERE $predicate;
     ''');
 
-    final scope = result.root.scope;
+    final scope = result.root.statementScope;
-    expect(scope.allOf<ResultSetAvailableInStatement>(), hasLength(2));
+    expect(scope.resultSets, hasLength(2));
   });
 }

sqlparser/test/analysis/column_resolver_test.dart

@@ -2,6 +2,7 @@ import 'package:sqlparser/sqlparser.dart';
 import 'package:test/test.dart';
 
 import 'data.dart';
+import 'errors/utils.dart';
 
 void main() {
   late SqlEngine engine;
@@ -238,4 +239,18 @@ INSERT INTO demo VALUES (?, ?)
       expect(root.resolvedTargetColumns, hasLength(1));
     });
   });
+
+  test('does not allow a subquery in from to read outer values', () {
+    final result = engine.analyze(
+        'SELECT * FROM demo d1, (SELECT * FROM demo i WHERE i.id = d1.id) d2;');
+
+    result.expectError('d1.id', type: AnalysisErrorType.referencedUnknownTable);
+  });
+
+  test('allows subquery expressions to read outer values', () {
+    final result = engine.analyze('SELECT * FROM demo d1 WHERE '
+        'EXISTS (SELECT * FROM demo i WHERE i.id = d1.id);');
+
+    result.expectNoError();
+  });
 }

sqlparser/test/analysis/reference_resolver_test.dart

@@ -4,6 +4,7 @@ import 'package:test/test.dart';
 
 import '../parser/utils.dart';
 import 'data.dart';
+import 'errors/utils.dart';
 
 void main() {
   test('correctly resolves return columns', () {
@@ -94,6 +95,15 @@ void main() {
     );
   });
 
+  test('does not allow references to result column outside of ORDER BY', () {
+    final engine = SqlEngine()..registerTable(demoTable);
+
+    final context = engine
+        .analyze('SELECT d.content, 3 * d.id AS t, t AS u FROM demo AS d');
+
+    context.expectError('t', type: AnalysisErrorType.referencedUnknownColumn);
+  });
+
   test('resolves columns from nested results', () {
     final engine = SqlEngine(EngineOptions(useDriftExtensions: true))
       ..registerTable(demoTable)
@@ -315,4 +325,39 @@ SELECT row_number() OVER wnd FROM demo
       testWith('DELETE FROM users RETURNING id;');
     });
   });
+
+  test('resolves column in foreign key declaration', () {
+    final engine = SqlEngine()..registerTableFromSql('''
+CREATE TABLE points (
+  id INTEGER NOT NULL PRIMARY KEY,
+  lat REAL NOT NULL,
+  long REAL NOT NULL
+);
+''');
+
+    final parseResult = engine.parse('''
+CREATE TABLE routes (
+  id INTEGER NOT NULL PRIMARY KEY,
+  "from" INTEGER NOT NULL REFERENCES points (id),
+  "to" INTEGER NOT NULL REFERENCES points (id)
+);
+''');
+    final table = const SchemaFromCreateTable()
+        .read(parseResult.rootNode as CreateTableStatement);
+    engine.registerTable(table);
+
+    final result = engine.analyzeParsed(parseResult);
+
+    result.expectNoError();
+
+    final createTable = result.root as CreateTableStatement;
+    final fromReference =
+        createTable.columns[1].constraints[1] as ForeignKeyColumnConstraint;
+    final fromReferenced =
+        fromReference.clause.columnNames.single.resolvedColumn;
+
+    expect(fromReferenced, isNotNull);
+    expect(
+        fromReferenced!.containingSet, result.rootScope.knownTables['points']);
+  });
 }

sqlparser/test/analysis/regression_test.dart

@@ -156,4 +156,36 @@ CREATE TABLE downloads (
             .having((e) => e.type, 'type', BasicType.int)
             .having((e) => e.nullable, 'nullable', isFalse));
   });
+
+  test('regression test for #1858', () {
+    // https://github.com/simolus3/drift/issues/1858
+    final engine = SqlEngine(
+        EngineOptions(useDriftExtensions: true, version: SqliteVersion.v3_38));
+
+    engine.registerTableFromSql('''
+CREATE TABLE IF NOT EXISTS contract_has_add_fees
+(
+   position_id INTEGER NOT NULL,
+   position_db INTEGER NOT NULL,
+   addfee      TEXT    NOT NULL,
+   amount      REAL    NOT NULL,
+   changed     DATETIME,
+   removed     BOOLEAN NOT NULL DEFAULT FALSE,
+   PRIMARY KEY (position_id, position_db, addfee)
+) WITHOUT ROWID;
+''');
+
+    final result = engine.analyze(r'''
+DELETE
+FROM contract_has_add_fees
+WHERE EXISTS(SELECT *
+             FROM json_each(:json) AS j
+             WHERE position_id = json_extract(j.value, '$.positionid')
+               AND position_db = json_extract(j.value, '$.positiondb')
+               AND addfee = json_extract(j.value, '$.addfee')
+         );
+''');
+
+    expect(result.errors, isEmpty);
+  });
 }

sqlparser/test/analysis/types2/join_analysis_test.dart

@@ -18,24 +18,18 @@ void main() {
 
     final model = JoinModel.of(stmt)!;
     expect(
-      model.isNullableTable(stmt.scope
+      model.isNullableTable(
-          .resolve<ResultSetAvailableInStatement>('a1')!
+          stmt.scope.resolveResultSet('a1')!.resultSet.resultSet!),
-          .resultSet
-          .resultSet!),
       isFalse,
     );
     expect(
-      model.isNullableTable(stmt.scope
+      model.isNullableTable(
-          .resolve<ResultSetAvailableInStatement>('a2')!
+          stmt.scope.resolveResultSet('a2')!.resultSet.resultSet!),
-          .resultSet
-          .resultSet!),
       isTrue,
     );
     expect(
-      model.isNullableTable(stmt.scope
+      model.isNullableTable(
-          .resolve<ResultSetAvailableInStatement>('a3')!
+          stmt.scope.resolveResultSet('a3')!.resultSet.resultSet!),
-          .resultSet
-          .resultSet!),
       isFalse,
     );
   });

sqlparser/test/utils/find_referenced_tables_test.dart

@@ -67,6 +67,7 @@ void main() {
         INSERT INTO logins (user, timestamp) VALUES (new.id, 0);
       END;
     ''');
+    expect(ctx.errors, isEmpty);
     final body = (ctx.root as CreateTriggerStatement).action;
 
     // Users referenced via "new" in body.