Add support for nested query variables

This commit is contained in:
Daniel Brauner 2022-01-18 22:21:54 +01:00
parent e99b66486d
commit d74860ec49
15 changed files with 417 additions and 74 deletions

View File

@ -0,0 +1,60 @@
import 'package:sqlparser/sqlparser.dart';
/// Generates additional rows in the select statement for the nested queries
class NestedQueryTransformer extends Transformer<void> {
AstNode rewrite(AstNode node) {
return transform(node, null)!;
}
@override
AstNode? visitSelectStatement(SelectStatement e, void arg) {
final collector = _NestedQueryVariableCollector();
e.accept(collector, null);
for (final result in collector.results) {
e.columns.add(
ExpressionResultColumn(
expression: Reference(
entityName: result.variable.entityName,
columnName: result.variable.columnName,
),
as: '${result.prefix}_${result.variable.name}',
),
);
}
// Only top level select statements support nested queries
return e;
}
}
class _NestedQueryVariableCollector extends RecursiveVisitor<String?, void> {
final List<_VariableWithPrefix> results;
_NestedQueryVariableCollector() : results = [];
@override
void visitMoorSpecificNode(MoorSpecificNode e, String? arg) {
if (e is NestedQueryColumn) {
super.visitMoorSpecificNode(e, e.queryName);
} else {
super.visitMoorSpecificNode(e, arg);
}
}
@override
void visitNestedQueryVariable(NestedQueryVariable e, String? arg) {
assert(arg != null, 'the query name should not be null here');
results.add(_VariableWithPrefix(arg!, e));
super.visitNestedQueryVariable(e, arg);
}
}
class _VariableWithPrefix {
final String prefix;
final NestedQueryVariable variable;
_VariableWithPrefix(this.prefix, this.variable);
}

View File

@ -7,6 +7,22 @@ import 'package:sqlparser/utils/find_referenced_tables.dart';
import 'lints/linter.dart';
import 'required_variables.dart';
/// The context contains all data that is required to create an [SqlQuery]. This
/// class is simply there to bundle the data.
class _QueryHandlerContext {
final List<FoundElement> foundElements;
final AstNode root;
final String queryName;
final String? requestedResultClass;
_QueryHandlerContext({
required List<FoundElement> foundElements,
required this.root,
required this.queryName,
this.requestedResultClass,
}) : foundElements = List.unmodifiable(foundElements);
}
/// Maps an [AnalysisContext] from the sqlparser to a [SqlQuery] from this
/// generator package by determining its type, return columns, variables and so
/// on.
@ -15,24 +31,29 @@ class QueryHandler {
final TypeMapper mapper;
final RequiredVariables requiredVariables;
/// Found tables and views found need to be shared between the query and
/// all subqueries to not muss any updates when watching query.
late Set<Table> _foundTables;
late Set<View> _foundViews;
late List<FoundElement> _foundElements;
Iterable<FoundVariable> get _foundVariables =>
_foundElements.whereType<FoundVariable>();
/// Used to create a unique name for every nested query. This needs to be
/// shared between queries, therefore this should not be part of the
/// context.
int nestedQueryCounter;
QueryHandler(
this.context,
this.mapper, {
this.requiredVariables = RequiredVariables.empty,
});
}) : nestedQueryCounter = 0;
SqlQuery handle(DeclaredQuery source) {
_foundElements =
mapper.extractElements(context, required: requiredVariables);
_verifyNoSkippedIndexes();
final foundElements = mapper.extractElements(
context,
context.root,
required: requiredVariables,
);
_verifyNoSkippedIndexes(foundElements);
final String? requestedResultClass;
if (source is DeclaredMoorQuery) {
@ -41,11 +62,12 @@ class QueryHandler {
requestedResultClass = null;
}
final query = _mapToMoor(
final query = _mapToMoor(_QueryHandlerContext(
foundElements: foundElements,
queryName: source.name,
requestedResultClass: requestedResultClass,
root: context.root,
);
));
final linter = Linter.forHandler(this);
linter.reportLints();
@ -54,23 +76,16 @@ class QueryHandler {
return query;
}
SqlQuery _mapToMoor({
required String queryName,
required String? requestedResultClass,
required AstNode root,
}) {
if (root is BaseSelectStatement) {
return _handleSelect(
queryName: queryName,
requestedResultClass: requestedResultClass,
select: root,
);
} else if (root is UpdateStatement ||
root is DeleteStatement ||
root is InsertStatement) {
return _handleUpdate(queryName, root);
SqlQuery _mapToMoor(_QueryHandlerContext queryContext) {
if (queryContext.root is BaseSelectStatement) {
return _handleSelect(queryContext);
} else if (queryContext.root is UpdateStatement ||
queryContext.root is DeleteStatement ||
queryContext.root is InsertStatement) {
return _handleUpdate(queryContext);
} else {
throw StateError('Unexpected sql: Got $root, expected insert, select, '
throw StateError(
'Unexpected sql: Got ${queryContext.root}, expected insert, select, '
'update or delete');
}
}
@ -80,7 +95,9 @@ class QueryHandler {
_foundViews = visitor.foundViews;
}
UpdatingQuery _handleUpdate(String queryName, AstNode root) {
UpdatingQuery _handleUpdate(_QueryHandlerContext queryContext) {
final root = queryContext.root;
final updatedFinder = UpdatedTablesVisitor();
root.acceptWithoutArg(updatedFinder);
_applyFoundTables(updatedFinder);
@ -91,15 +108,15 @@ class QueryHandler {
if (root is StatementReturningColumns) {
final columns = root.returnedResultSet?.resolvedColumns;
if (columns != null) {
resultSet = _inferResultSet(root, columns);
resultSet = _inferResultSet(queryContext, columns);
}
}
return UpdatingQuery(
queryName,
queryContext.queryName,
context,
root,
_foundElements,
queryContext.foundElements,
updatedFinder.writtenTables
.map(mapper.writtenToMoor)
.whereType<WrittenMoorTable>()
@ -110,14 +127,10 @@ class QueryHandler {
);
}
SqlSelectQuery _handleSelect({
required String queryName,
required String? requestedResultClass,
required BaseSelectStatement select,
}) {
SqlSelectQuery _handleSelect(_QueryHandlerContext queryContext) {
final tableFinder = ReferencedTablesVisitor();
select.acceptWithoutArg(tableFinder);
// fine
queryContext.root.acceptWithoutArg(tableFinder);
_applyFoundTables(tableFinder);
final moorTables =
@ -128,17 +141,23 @@ class QueryHandler {
final moorEntities = [...moorTables, ...moorViews];
return SqlSelectQuery(
queryName,
queryContext.queryName,
context,
select,
_foundElements,
queryContext.root,
queryContext.foundElements,
moorEntities,
_inferResultSet(select, select.resolvedColumns!),
requestedResultClass,
_inferResultSet(
queryContext,
(queryContext.root as SelectStatement).resolvedColumns!,
),
queryContext.requestedResultClass,
);
}
InferredResultSet _inferResultSet(AstNode select, List<Column> rawColumns) {
InferredResultSet _inferResultSet(
_QueryHandlerContext queryContext,
List<Column> rawColumns,
) {
final candidatesForSingleTable = {..._foundTables, ..._foundViews};
final columns = <ResultColumn>[];
@ -158,7 +177,7 @@ class QueryHandler {
candidatesForSingleTable.removeWhere((t) => t != resultSet);
}
final nestedResults = _findNestedResultTables(select);
final nestedResults = _findNestedResultTables(queryContext);
if (nestedResults.isNotEmpty) {
// The single table optimization doesn't make sense when nested result
// sets are present.
@ -220,9 +239,11 @@ class QueryHandler {
return InferredResultSet(null, columns, nestedResults: nestedResults);
}
List<NestedResult> _findNestedResultTables(AstNode query) {
List<NestedResult> _findNestedResultTables(
_QueryHandlerContext queryContext) {
// We don't currently support nested results for compound statements
if (query is! SelectStatement) return const [];
if (queryContext.root is! SelectStatement) return const [];
final query = queryContext.root as SelectStatement;
final nestedTables = <NestedResult>[];
final analysis = JoinModel.of(query);
@ -243,13 +264,24 @@ class QueryHandler {
isNullable: isNullable,
));
} else if (column is NestedQueryColumn) {
final foundElements = mapper.extractElements(
context,
column.select,
required: requiredVariables,
);
_verifyNoSkippedIndexes(foundElements);
final name = 'nested_query_${nestedQueryCounter++}';
column.queryName = name;
nestedTables.add(NestedResultQuery(
from: column,
query: _handleSelect(
queryName: 'nested',
query: _handleSelect(_QueryHandlerContext(
queryName: name,
requestedResultClass: column.as,
select: column.select,
),
root: column.select,
foundElements: foundElements,
)),
));
}
}
@ -295,8 +327,8 @@ class QueryHandler {
/// We verify that no variable numbers are skipped in the query. For instance,
/// `SELECT * FROM tbl WHERE a = ?2 AND b = ?` would fail this check because
/// the index 1 is never used.
void _verifyNoSkippedIndexes() {
final variables = List.of(_foundVariables)
void _verifyNoSkippedIndexes(List<FoundElement> foundElements) {
final variables = List.of(foundElements.whereType<FoundVariable>())
..sort((a, b) => a.index.compareTo(b.index));
var currentExpectedIndex = 1;

View File

@ -120,14 +120,17 @@ class TypeMapper {
/// a Dart placeholder, its indexed is LOWER than that element. This means
/// that elements can be expanded into multiple variables without breaking
/// variables that appear after them.
List<FoundElement> extractElements(AnalysisContext ctx,
List<FoundElement> extractElements(AnalysisContext ctx, AstNode root,
{RequiredVariables required = RequiredVariables.empty}) {
// this contains variable references. For instance, SELECT :a = :a would
// contain two entries, both referring to the same variable. To do that,
// we use the fact that each variable has a unique index.
final variables = ctx.root.allDescendants.whereType<Variable>().toList();
final collector = _VariableCollector();
root.accept(collector, null);
final variables = collector.result;
final placeholders =
ctx.root.allDescendants.whereType<DartPlaceholder>().toList();
root.allDescendants.whereType<DartPlaceholder>().toList();
final merged = _mergeVarsAndPlaceholders(variables, placeholders);
@ -152,6 +155,18 @@ class TypeMapper {
(used is NumberedVariable) ? used.explicitIndex : null;
final internalType = ctx.typeOf(used);
final type = resolvedToMoor(internalType.type);
if (used is NestedQueryVariable) {
foundElements.add(FoundVariable.nestedQuery(
index: currentIndex,
name: used.name,
type: type,
variable: used,
));
continue;
}
final isArray = internalType.type?.isArray ?? false;
final isRequired = required.requiredNamedVariables.contains(name) ||
required.requiredNumberedVariables.contains(used.resolvedIndex);
@ -340,3 +355,27 @@ class TypeMapper {
}
}
}
/// Because nested variables should not be included when extracting all
/// FoundElements allDescendants.whereType<Variable>() no longer works.
class _VariableCollector extends RecursiveVisitor<void, void> {
final List<Variable> result;
_VariableCollector() : result = [];
@override
void visitVariable(Variable e, void arg) {
result.add(e);
super.visitVariable(e, arg);
}
@override
void visitMoorSpecificNode(MoorSpecificNode e, void arg) {
if (e is NestedQueryColumn) {
return;
}
super.visitMoorSpecificNode(e, arg);
}
}

View File

@ -158,6 +158,28 @@ abstract class SqlQuery {
return resultClassName;
}
/// Returns all found elements, from this query an all nested queries. The
/// elements returned by this method are in no particular order, thus they
/// can only be used to determine the method parameters.
///
/// This method makes some effort to remove duplicated parameters. But only
/// by comparing the dart name.
List<FoundElement> elementsWithNestedQueries() {
final elements = List.of(this.elements);
final subQueries = resultSet?.nestedResults.whereType<NestedResultQuery>();
for (final subQuery in subQueries ?? const <NestedResultQuery>[]) {
for (final subElement in subQuery.query.elementsWithNestedQueries()) {
if (elements
.none((e) => e.dartParameterName == subElement.dartParameterName)) {
elements.add(subElement);
}
}
}
return elements;
}
}
class SqlSelectQuery extends SqlQuery {
@ -539,6 +561,9 @@ abstract class FoundElement {
bool get hasSqlName => name != null;
/// If the element should be hidden from the parameter list
bool get hidden => false;
/// Dart code for a type representing tis element.
String dartTypeCode([GenerationOptions options = const GenerationOptions()]);
}
@ -581,6 +606,12 @@ class FoundVariable extends FoundElement implements HasType {
final bool isRequired;
@override
final bool hidden;
/// Whether this variable is used as input for a nested query or not.
final bool nestedQuery;
FoundVariable({
required this.index,
required this.name,
@ -590,7 +621,21 @@ class FoundVariable extends FoundElement implements HasType {
this.isArray = false,
this.isRequired = false,
this.typeConverter,
}) : assert(variable.resolvedIndex == index);
}) : hidden = false,
nestedQuery = false,
assert(variable.resolvedIndex == index);
FoundVariable.nestedQuery({
required this.index,
required this.name,
required this.type,
required this.variable,
}) : typeConverter = null,
nullable = false,
isArray = false,
isRequired = true,
nestedQuery = true,
hidden = true;
@override
String get dartParameterName {

View File

@ -3,6 +3,7 @@ import 'dart:math' show max;
import 'package:drift_dev/moor_generator.dart';
import 'package:drift_dev/src/analyzer/options.dart';
import 'package:drift_dev/src/analyzer/sql_queries/explicit_alias_transformer.dart';
import 'package:drift_dev/src/analyzer/sql_queries/nested_query_transformer.dart';
import 'package:drift_dev/src/utils/string_escaper.dart';
import 'package:drift_dev/writer.dart';
import 'package:recase/recase.dart';
@ -44,8 +45,8 @@ class QueryWriter {
// We do this transformation so late because it shouldn't have an impact on
// analysis, Dart getter names stay the same.
if (resultSet != null && options.newSqlCodeGeneration) {
_transformer = ExplicitAliasTransformer();
_transformer.rewrite(query.root!);
ExplicitAliasTransformer().rewrite(query.root!);
NestedQueryTransformer().rewrite(query.root!);
}
if (query is SqlSelectQuery) {
@ -309,7 +310,9 @@ class QueryWriter {
}
var needsComma = false;
for (final element in query.elements) {
for (final element in query.elementsWithNestedQueries()) {
if (element.hidden) continue;
// Placeholders with a default value generate optional (and thus, named)
// parameters. Since moor 4, we have an option to also generate named
// parameters for named variables.
@ -860,6 +863,8 @@ class _ExpandedVariableWriter {
if (needsNullAssertion) {
buffer.write('!');
}
} else if (element.nestedQuery) {
buffer.write('row.read(\'${query.name}_$dartExpr\')');
} else {
buffer.write(dartExpr);
}

View File

@ -12,6 +12,7 @@ export 'types/types.dart' show TypeInferenceResults;
part 'context.dart';
part 'error.dart';
part 'options.dart';
part 'schema/column.dart';
part 'schema/from_create_table.dart';
part 'schema/references.dart';
@ -20,10 +21,10 @@ part 'schema/table.dart';
part 'schema/view.dart';
part 'steps/column_resolver.dart';
part 'steps/linting_visitor.dart';
part 'steps/nested_query_resolver.dart';
part 'steps/prepare_ast.dart';
part 'steps/reference_resolver.dart';
part 'steps/set_parent_visitor.dart';
part 'options.dart';
part 'utils/expand_function_parameters.dart';
/// Something that can be represented in a human-readable description.

View File

@ -97,7 +97,14 @@ class ReferenceScope {
/// Resolves to a [Referencable] with the given [name] and of the type [T].
/// If the reference couldn't be found, null is returned and [orElse] will be
/// called.
T? resolve<T extends Referencable>(String name, {Function()? orElse}) {
///
/// If [includeParents] is set to false, resolve will only search the current
/// scope.
T? resolve<T extends Referencable>(
String name, {
Function()? orElse,
bool includeParents = true,
}) {
ReferenceScope? scope = this;
var isAtParent = false;
final upper = name.toUpperCase();
@ -115,6 +122,8 @@ class ReferenceScope {
scope = scope.parent;
isAtParent = true;
if (!includeParents) break;
}
if (orElse != null) orElse();

View File

@ -0,0 +1,68 @@
part of '../analysis.dart';
/// Converts all references in nested queries, that require data from the
/// parent query into variables.
class NestedQueryResolver extends RecursiveVisitor<void, void> {
final AnalysisContext context;
NestedQueryResolver(this.context);
@override
void visitMoorSpecificNode(MoorSpecificNode e, void arg) {
if (e is NestedQueryColumn) {
_transform(context, e);
} else {
super.visitMoorSpecificNode(e, arg);
}
}
}
void _transform(AnalysisContext context, NestedQueryColumn e) {
e.select.transformChildren(_NestedQueryTransformer(context), null);
AstPreparingVisitor.resolveIndexOfVariables(
e.allDescendants.whereType<Variable>().toList(),
);
}
class _NestedQueryTransformer extends Transformer<void> {
final AnalysisContext context;
_NestedQueryTransformer(this.context);
@override
AstNode? visitReference(Reference e, void arg) {
// if the scope of the nested query cannot resolve the reference, the
// reference needs to be retrieved from the parent query
if (e.entityName != null &&
e.scope.resolve(e.entityName!, includeParents: false) == null) {
final result = e.scope.resolve(e.entityName!);
if (result == null) {
context.reportError(AnalysisError(
type: AnalysisErrorType.referencedUnknownTable,
message: 'Unknown table or view in nested query: ${e.entityName}',
relevantNode: e,
));
} else {
return NestedQueryVariable(
entityName: e.entityName,
columnName: e.columnName,
)..setSpan(e.first!, e.last!);
}
}
return super.visitReference(e, arg);
}
@override
AstNode? visitMoorSpecificNode(MoorSpecificNode e, void arg) {
if (e is NestedQueryColumn) {
_transform(context, e);
return e;
} else {
super.visitMoorSpecificNode(e, arg);
}
}
}

View File

@ -16,7 +16,7 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
void start(AstNode root) {
root.accept(this, null);
_resolveIndexOfVariables();
resolveIndexOfVariables(_foundVariables);
}
@override
@ -170,16 +170,22 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
visitChildren(e, arg);
}
void _resolveIndexOfVariables() {
@override
void visitNestedQueryVariable(NestedQueryVariable e, void arg) {
_foundVariables.add(e);
visitChildren(e, arg);
}
static void resolveIndexOfVariables(List<Variable> variables) {
// sort variables by the order in which they appear inside the statement.
_foundVariables.sort((a, b) {
variables.sort((a, b) {
return a.firstPosition.compareTo(b.firstPosition);
});
// Assigning rules are explained at https://www.sqlite.org/lang_expr.html#varparam
var largestAssigned = 0;
final resolvedNames = <String, int>{};
for (final variable in _foundVariables) {
for (final variable in variables) {
if (variable is NumberedVariable) {
// if the variable has an explicit index (e.g ?123), then 123 is the
// resolved index and the next variable will have index 124. Otherwise,
@ -249,4 +255,18 @@ class AstPreparingVisitor extends RecursiveVisitor<void, void> {
visitChildren(e, null);
}
/// If a nested query was found. Collect everything separately.
@override
void visitMoorSpecificNode(MoorSpecificNode e, void arg) {
if (e is NestedQueryColumn) {
// create a new scope for the nested query to differentiate between
// references that can be resolved in the nested query and references
// which require data from the parent query
e.select.scope = e.scope.createChild();
AstPreparingVisitor(context: context).start(e.select);
} else {
super.visitMoorSpecificNode(e, arg);
}
}
}

View File

@ -29,13 +29,14 @@ class NumberedVariable extends Expression implements Variable {
}
class ColonNamedVariable extends Expression implements Variable {
final ColonVariableToken token;
String get name => token.name;
final String name;
@override
int? resolvedIndex;
ColonNamedVariable(this.token);
ColonNamedVariable._(this.name);
ColonNamedVariable(ColonVariableToken token) : name = token.name;
@override
R accept<A, R>(AstVisitor<A, R> visitor, A arg) {
@ -48,3 +49,38 @@ class ColonNamedVariable extends Expression implements Variable {
@override
Iterable<AstNode> get childNodes => [];
}
/// A variable that is created when a nested query requires data from the
/// main query. In most cases this can be treated like a colon named
/// variable, this this extends [ColonNamedVariable].
class NestedQueryVariable extends ColonNamedVariable {
static String _nameFrom(String? entityName, String? columnName) {
final buf = StringBuffer();
if (entityName != null) {
buf.write('${entityName}_');
}
buf.write(columnName);
return buf.toString();
}
final String? entityName;
final String columnName;
NestedQueryVariable({
required this.entityName,
required this.columnName,
}) : super._(_nameFrom(entityName, columnName));
@override
R accept<A, R>(AstVisitor<A, R> visitor, A arg) {
return visitor.visitNestedQueryVariable(this, arg);
}
@override
Iterable<AstNode> get childNodes => const [];
@override
void transformChildren<A>(Transformer<A> transformer, A arg) {}
}

View File

@ -1,10 +1,17 @@
import '../../analysis/analysis.dart';
import '../ast.dart'
show StarResultColumn, ResultColumn, Renamable, SelectStatement;
import '../node.dart';
import '../visitor.dart';
import 'moor_file.dart';
/// To wrap the query name into its own type, to avoid conflicts when using
/// the [AstNode] metadata.
class _NestedColumnNameMetadata {
final String? name;
_NestedColumnNameMetadata(this.name);
}
/// A nested query column, denoted by `LIST(...)` in user queries.
///
/// Nested query columns take a select query and execute it for every result
@ -12,7 +19,7 @@ import 'moor_file.dart';
/// top level select query, because the result of them can only be computed
/// in dart.
class NestedQueryColumn extends ResultColumn
implements MoorSpecificNode, Renamable, Referencable {
implements MoorSpecificNode, Renamable {
@override
final String? as;
@ -33,7 +40,9 @@ class NestedQueryColumn extends ResultColumn
return visitor.visitMoorSpecificNode(this, arg);
}
// idk is this required?
@override
bool get visibleToChildren => false;
/// The unique name for this query. Used to identify it and it's variables in
/// the AST tree.
set queryName(String? name) => setMeta(_NestedColumnNameMetadata(name));
String? get queryName => meta<_NestedColumnNameMetadata>()?.name;
}

View File

@ -88,6 +88,7 @@ abstract class AstVisitor<A, R> {
R visitNumberedVariable(NumberedVariable e, A arg);
R visitNamedVariable(ColonNamedVariable e, A arg);
R visitNestedQueryVariable(NestedQueryVariable e, A arg);
R visitBlock(Block block, A arg);
R visitBeginTransaction(BeginTransactionStatement e, A arg);
@ -542,6 +543,11 @@ class RecursiveVisitor<A, R> implements AstVisitor<A, R?> {
return visitVariable(e, arg);
}
@override
R? visitNestedQueryVariable(NestedQueryVariable e, A arg) {
return visitVariable(e, arg);
}
R? visitVariable(Variable e, A arg) {
return visitExpression(e, arg);
}

View File

@ -201,7 +201,8 @@ class SqlEngine {
node
..acceptWithoutArg(ColumnResolver(context))
..acceptWithoutArg(ReferenceResolver(context));
..acceptWithoutArg(ReferenceResolver(context))
..acceptWithoutArg(NestedQueryResolver(context));
final session = TypeInferenceSession(context, options);
final resolver = TypeResolver(session);

View File

@ -501,6 +501,13 @@ class EqualityEnforcingVisitor implements AstVisitor<void, void> {
_checkChildren(e);
}
@override
void visitNestedQueryVariable(NestedQueryVariable e, void arg) {
final current = _currentAs<NestedQueryVariable>(e);
_assert(current.name == e.name, e);
_checkChildren(e);
}
@override
void visitNullLiteral(NullLiteral e, void arg) {
_currentAs<NullLiteral>(e);

View File

@ -834,6 +834,11 @@ class NodeSqlBuilder extends AstVisitor<void, void> {
symbol(e.name, spaceBefore: true, spaceAfter: true);
}
@override
void visitNestedQueryVariable(NestedQueryVariable e, void arg) {
visitNamedVariable(e, arg);
}
@override
void visitNullLiteral(NullLiteral e, void arg) {
_keyword(TokenType.$null);