From 181ec2f22f93c07aa5cc7262177fdf5e33079a02 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:25:55 -0700 Subject: [PATCH 1/4] Follow the outer side of two-input nodes through an abstract type get_source and _collect_nodes! needed a method per node type that combines two sources, because those carry `outer`/`inner` rather than `source`. With outer joins, seven set operators and zip arriving, that is eleven more pairs of near-identical methods. Introduce QueryableBinary instead, and subtype QueryableJoin and QueryableGroupJoin under it. The tree walk now has one method covering every two-input node, and a new one only has to pick the right supertype. QueryableBinary <: Queryable and the concrete types are unchanged, so `node isa QueryableJoin` and anything dispatching on Queryable still behave exactly as before. Co-Authored-By: Claude Opus 5 --- src/query_tree.jl | 16 ++++------------ src/queryable/queryable.jl | 10 ++++++++++ src/queryable/queryable_groupjoin.jl | 2 +- src/queryable/queryable_join.jl | 2 +- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/query_tree.jl b/src/query_tree.jl index 703764e..8d6cb97 100644 --- a/src/query_tree.jl +++ b/src/query_tree.jl @@ -1,17 +1,14 @@ """ get_source(q::Queryable) -Follow the `.source` (or `.outer`) chain to the root `QueryableSource`. +Follow the `.source` (or, for two-input nodes, `.outer`) chain to the root +`QueryableSource`. """ function get_source(q::QueryableSource) return q end -function get_source(q::QueryableJoin) - return get_source(q.outer) -end - -function get_source(q::QueryableGroupJoin) +function get_source(q::QueryableBinary) return get_source(q.outer) end @@ -34,12 +31,7 @@ function _collect_nodes!(nodes, q::QueryableSource) push!(nodes, q) end -function _collect_nodes!(nodes, q::QueryableJoin) - push!(nodes, q) - _collect_nodes!(nodes, q.outer) -end - -function _collect_nodes!(nodes, q::QueryableGroupJoin) +function _collect_nodes!(nodes, q::QueryableBinary) push!(nodes, q) _collect_nodes!(nodes, q.outer) end diff --git a/src/queryable/queryable.jl b/src/queryable/queryable.jl index b13c294..82312c9 100644 --- a/src/queryable/queryable.jl +++ b/src/queryable/queryable.jl @@ -1,5 +1,15 @@ abstract type Queryable end +""" + QueryableBinary <: Queryable + +Nodes that combine two sources — joins, set operations, `zip`. They carry +`outer` and `inner` fields, and the tree walk follows `outer`, so the query +plan reads as a chain hanging off the primary input with the secondary input +attached at the node. +""" +abstract type QueryableBinary <: Queryable end + QueryOperators.query(x::Queryable) = x IteratorInterfaceExtensions.isiterable(x::Queryable) = true diff --git a/src/queryable/queryable_groupjoin.jl b/src/queryable/queryable_groupjoin.jl index 6dfd0f1..33baa49 100644 --- a/src/queryable/queryable_groupjoin.jl +++ b/src/queryable/queryable_groupjoin.jl @@ -1,4 +1,4 @@ -struct QueryableGroupJoin <: Queryable +struct QueryableGroupJoin <: QueryableBinary outer inner outerKeySelector_func diff --git a/src/queryable/queryable_join.jl b/src/queryable/queryable_join.jl index 4dde26e..dab1a61 100644 --- a/src/queryable/queryable_join.jl +++ b/src/queryable/queryable_join.jl @@ -1,4 +1,4 @@ -struct QueryableJoin <: Queryable +struct QueryableJoin <: QueryableBinary outer inner outerKeySelector_func From f781a67c4f8bdd9d16609efa5991e3dc0f6dbec1 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:25:55 -0700 Subject: [PATCH 2/4] Add Queryable nodes for the new LINQ operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a node per operator introduced in QueryOperators 1.2, so a backend sees them in the query tree rather than having the in-memory implementation run behind its back: the outer joins, the seven set operators, the four partitioning operators, reverse/shuffle/index, append/prepend/zip, count_by/aggregate_by/chunk, and of_type/cast. Each gets a describe_node method so @queryplan() renders it. order and order_descending add no node — they lower to orderby with an identity key selector, so thenby can still follow them and a backend can recognise the identity selector and sort by every column. The plain and _by forms of the set operators share a node, with a `nothing` key selector marking the plain form; zip does the same with its result selector. Terminal operators needed new machinery. The tree had no notion of an operator that returns a value, and QueryOperators.count had no method taking a Queryable at all, so a query ending in @count() failed outright. QueryableScalar now captures one, and execute_scalar dispatches on the root source type so a backend can specialise _execute_scalar and translate the operator; the fallback materialises the source and delegates to the in-memory implementation, which every backend gets for free. count is fixed as a side effect, and a regression test pins it. Requires QueryOperators 1.2, which is not registered yet, so CI here stays red until it is. Co-Authored-By: Claude Opus 5 --- Project.toml | 4 +- src/QueryableBackend.jl | 8 + src/query_plan.jl | 59 ++++++ src/queryable/queryable_combining.jl | 37 ++++ src/queryable/queryable_keyedaggregation.jl | 33 ++++ src/queryable/queryable_ordering.jl | 43 ++++ src/queryable/queryable_outerjoin.jl | 43 ++++ src/queryable/queryable_partition.jl | 41 ++++ src/queryable/queryable_scalar.jl | 103 ++++++++++ src/queryable/queryable_setops.jl | 47 +++++ src/queryable/queryable_typefiltering.jl | 19 ++ test/runtests.jl | 1 + test/test_new_operators.jl | 209 ++++++++++++++++++++ 13 files changed, 645 insertions(+), 2 deletions(-) create mode 100644 src/queryable/queryable_combining.jl create mode 100644 src/queryable/queryable_keyedaggregation.jl create mode 100644 src/queryable/queryable_ordering.jl create mode 100644 src/queryable/queryable_outerjoin.jl create mode 100644 src/queryable/queryable_partition.jl create mode 100644 src/queryable/queryable_scalar.jl create mode 100644 src/queryable/queryable_setops.jl create mode 100644 src/queryable/queryable_typefiltering.jl create mode 100644 test/test_new_operators.jl diff --git a/Project.toml b/Project.toml index ff953bd..9ad87e6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "QueryableBackend" uuid = "0898d9ac-042b-5d31-8dcc-959fb3365f19" -version = "1.0.1-DEV" +version = "1.1.0-DEV" [deps] QueryOperators = "2aef5ad7-51ca-5a8f-8e88-e75cf067b44b" @@ -14,7 +14,7 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] julia = "1" -QueryOperators = "0.8, 0.9, 1" +QueryOperators = "1.2" IteratorInterfaceExtensions = "1" TableTraits = "1" diff --git a/src/QueryableBackend.jl b/src/QueryableBackend.jl index 08af8f0..cc61851 100644 --- a/src/QueryableBackend.jl +++ b/src/QueryableBackend.jl @@ -10,13 +10,21 @@ include("queryable/queryable_orderby.jl") include("queryable/queryable_thenby.jl") include("queryable/queryable_join.jl") include("queryable/queryable_groupjoin.jl") +include("queryable/queryable_outerjoin.jl") include("queryable/queryable_mapmany.jl") include("queryable/queryable_take.jl") include("queryable/queryable_drop.jl") include("queryable/queryable_unique.jl") +include("queryable/queryable_setops.jl") +include("queryable/queryable_partition.jl") +include("queryable/queryable_ordering.jl") +include("queryable/queryable_combining.jl") +include("queryable/queryable_keyedaggregation.jl") +include("queryable/queryable_typefiltering.jl") include("source_queryable.jl") include("query_tree.jl") +include("queryable/queryable_scalar.jl") include("query_plan.jl") end diff --git a/src/query_plan.jl b/src/query_plan.jl index 655ac42..6f98777 100644 --- a/src/query_plan.jl +++ b/src/query_plan.jl @@ -71,6 +71,65 @@ function describe_node(q::QueryableMapMany) return "MapMany: $(string(q.collectionSelector_expr)) => $(string(q.resultSelector_expr))" end +function describe_node(q::QueryableLeftJoin) + return "LeftJoin: $(string(q.outerKeySelector_expr)) = $(string(q.innerKeySelector_expr)) => $(string(q.resultSelector_expr))" +end + +function describe_node(q::QueryableRightJoin) + return "RightJoin: $(string(q.outerKeySelector_expr)) = $(string(q.innerKeySelector_expr)) => $(string(q.resultSelector_expr))" +end + +function describe_node(q::QueryableFullJoin) + return "FullJoin: $(string(q.outerKeySelector_expr)) = $(string(q.innerKeySelector_expr)) => $(string(q.resultSelector_expr))" +end + +describe_node(::QueryableConcat) = "Concat" + +describe_node(q::QueryableUnion) = _describe_setop("Union", q) + +describe_node(q::QueryableExcept) = _describe_setop("Except", q) + +describe_node(q::QueryableIntersect) = _describe_setop("Intersect", q) + +# A `nothing` key selector is the plain form, which compares whole elements. +function _describe_setop(name, q) + return q.f_expr === nothing ? name : "$(name)By: $(string(q.f_expr))" +end + +describe_node(q::QueryableTakeWhile) = "TakeWhile: $(string(q.f_expr))" + +describe_node(q::QueryableDropWhile) = "DropWhile: $(string(q.f_expr))" + +describe_node(q::QueryableTakeLast) = "TakeLast: $(q.n)" + +describe_node(q::QueryableDropLast) = "DropLast: $(q.n)" + +describe_node(::QueryableReverse) = "Reverse" + +describe_node(::QueryableShuffle) = "Shuffle" + +describe_node(::QueryableIndex) = "Index" + +describe_node(q::QueryableAppend) = "Append: $(repr(q.element))" + +describe_node(q::QueryablePrepend) = "Prepend: $(repr(q.element))" + +function describe_node(q::QueryableZip) + return q.resultSelector_expr === nothing ? "Zip" : "Zip: $(string(q.resultSelector_expr))" +end + +describe_node(q::QueryableCountBy) = "CountBy: $(string(q.f_expr))" + +describe_node(q::QueryableAggregateBy) = "AggregateBy: $(string(q.f_expr))" + +describe_node(q::QueryableChunk) = "Chunk: $(q.n)" + +describe_node(q::QueryableOfType) = "OfType: $(q.T)" + +describe_node(q::QueryableCast) = "Cast: $(q.T)" + +describe_node(q::QueryableScalar) = "Scalar: $(q.op)" + # Fallback for unknown/extension types describe_node(q::Queryable) = string(typeof(q)) diff --git a/src/queryable/queryable_combining.jl b/src/queryable/queryable_combining.jl new file mode 100644 index 0000000..a8e7aa8 --- /dev/null +++ b/src/queryable/queryable_combining.jl @@ -0,0 +1,37 @@ +struct QueryableAppend <: Queryable + source + element + getiterator +end + +struct QueryablePrepend <: Queryable + source + element + getiterator +end + +struct QueryableZip <: QueryableBinary + outer + inner + resultSelector_func + resultSelector_expr + getiterator +end + +function QueryOperators.append(source::Queryable, element) + return QueryableAppend(source, element, source.getiterator) +end + +function QueryOperators.prepend(source::Queryable, element) + return QueryablePrepend(source, element, source.getiterator) +end + +# As with the set operators, a `nothing` result selector means the default: +# pair the two elements into a tuple. +function QueryOperators.zip(outer::Queryable, inner) + return QueryableZip(outer, inner, nothing, nothing, outer.getiterator) +end + +function QueryOperators.zip(outer::Queryable, inner, f_resultSelector::Function, resultSelector::Expr) + return QueryableZip(outer, inner, f_resultSelector, resultSelector, outer.getiterator) +end diff --git a/src/queryable/queryable_keyedaggregation.jl b/src/queryable/queryable_keyedaggregation.jl new file mode 100644 index 0000000..75b63fc --- /dev/null +++ b/src/queryable/queryable_keyedaggregation.jl @@ -0,0 +1,33 @@ +struct QueryableCountBy <: Queryable + source + f_func + f_expr + getiterator +end + +struct QueryableAggregateBy <: Queryable + source + f_func + f_expr + seed + accumulator_func + getiterator +end + +struct QueryableChunk <: Queryable + source + n::Int + getiterator +end + +function QueryOperators.count_by(source::Queryable, f::Function, f_expr::Expr) + return QueryableCountBy(source, f, f_expr, source.getiterator) +end + +function QueryOperators.aggregate_by(source::Queryable, f::Function, f_expr::Expr, seed, accumulator::Function) + return QueryableAggregateBy(source, f, f_expr, seed, accumulator, source.getiterator) +end + +function QueryOperators.chunk(source::Queryable, n::Integer) + return QueryableChunk(source, Int(n), source.getiterator) +end diff --git a/src/queryable/queryable_ordering.jl b/src/queryable/queryable_ordering.jl new file mode 100644 index 0000000..adbfba2 --- /dev/null +++ b/src/queryable/queryable_ordering.jl @@ -0,0 +1,43 @@ +struct QueryableReverse <: Queryable + source + getiterator +end + +struct QueryableShuffle <: Queryable + source + rng + getiterator +end + +struct QueryableIndex <: Queryable + source + getiterator +end + +# `order` and `order_descending` sort by the elements themselves, which is an +# orderby with an identity key selector. They reuse QueryableOrderBy rather +# than adding a node type, so `thenby` can still follow them and a backend can +# recognise the identity selector to sort by every column. +function QueryOperators.order(source::Queryable) + return QueryOperators.orderby(source, identity, :(i -> i)) +end + +function QueryOperators.order_descending(source::Queryable) + return QueryOperators.orderby_descending(source, identity, :(i -> i)) +end + +function QueryOperators.reverse(source::Queryable) + return QueryableReverse(source, source.getiterator) +end + +function QueryOperators.shuffle(source::Queryable) + return QueryableShuffle(source, nothing, source.getiterator) +end + +function QueryOperators.shuffle(source::Queryable, rng) + return QueryableShuffle(source, rng, source.getiterator) +end + +function QueryOperators.index(source::Queryable) + return QueryableIndex(source, source.getiterator) +end diff --git a/src/queryable/queryable_outerjoin.jl b/src/queryable/queryable_outerjoin.jl new file mode 100644 index 0000000..a3c271e --- /dev/null +++ b/src/queryable/queryable_outerjoin.jl @@ -0,0 +1,43 @@ +struct QueryableLeftJoin <: QueryableBinary + outer + inner + outerKeySelector_func + outerKeySelector_expr + innerKeySelector_func + innerKeySelector_expr + resultSelector_func + resultSelector_expr + getiterator +end + +struct QueryableRightJoin <: QueryableBinary + outer + inner + outerKeySelector_func + outerKeySelector_expr + innerKeySelector_func + innerKeySelector_expr + resultSelector_func + resultSelector_expr + getiterator +end + +struct QueryableFullJoin <: QueryableBinary + outer + inner + outerKeySelector_func + outerKeySelector_expr + innerKeySelector_func + innerKeySelector_expr + resultSelector_func + resultSelector_expr + getiterator +end + +for (op, node) in ((:left_join, :QueryableLeftJoin), + (:right_join, :QueryableRightJoin), + (:full_join, :QueryableFullJoin)) + @eval function QueryOperators.$op(outer::Queryable, inner, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + return $node(outer, inner, f_outerKeySelector, outerKeySelector, f_innerKeySelector, innerKeySelector, f_resultSelector, resultSelector, outer.getiterator) + end +end diff --git a/src/queryable/queryable_partition.jl b/src/queryable/queryable_partition.jl new file mode 100644 index 0000000..b2e73de --- /dev/null +++ b/src/queryable/queryable_partition.jl @@ -0,0 +1,41 @@ +struct QueryableTakeWhile <: Queryable + source + f_func + f_expr + getiterator +end + +struct QueryableDropWhile <: Queryable + source + f_func + f_expr + getiterator +end + +struct QueryableTakeLast <: Queryable + source + n::Int + getiterator +end + +struct QueryableDropLast <: Queryable + source + n::Int + getiterator +end + +function QueryOperators.take_while(source::Queryable, f::Function, f_expr::Expr) + return QueryableTakeWhile(source, f, f_expr, source.getiterator) +end + +function QueryOperators.drop_while(source::Queryable, f::Function, f_expr::Expr) + return QueryableDropWhile(source, f, f_expr, source.getiterator) +end + +function QueryOperators.take_last(source::Queryable, n::Integer) + return QueryableTakeLast(source, Int(n), source.getiterator) +end + +function QueryOperators.drop_last(source::Queryable, n::Integer) + return QueryableDropLast(source, Int(n), source.getiterator) +end diff --git a/src/queryable/queryable_scalar.jl b/src/queryable/queryable_scalar.jl new file mode 100644 index 0000000..6f80df3 --- /dev/null +++ b/src/queryable/queryable_scalar.jl @@ -0,0 +1,103 @@ +""" + QueryableScalar <: Queryable + +A terminal operator — one that returns a value rather than another sequence. + +Unlike every other node this is never left sitting in a tree: the +`QueryOperators` method that builds it immediately hands it to +[`execute_scalar`](@ref). It exists so that a backend has something to inspect +when it wants to push the operator down instead of materialising the source. +""" +struct QueryableScalar <: Queryable + source + op::Symbol + f + args::Tuple + getiterator +end + +""" + execute_scalar(q::QueryableScalar) + +Produce the value of a terminal operator. + +Dispatches on the type of the root source, so a backend can specialise +`_execute_scalar` for its own source type and translate the operator instead. +The fallback materialises the source and delegates to the in-memory +`QueryOperators` implementation, which every backend gets for free. +""" +execute_scalar(q::QueryableScalar) = _execute_scalar(get_source(q.source), q) + +function _execute_scalar(::QueryableSource, q::QueryableScalar) + return q.f(_materialize(q.source), q.args...) +end + +_materialize(source) = QueryOperators.query(IteratorInterfaceExtensions.getiterator(source)) + +# Builds the node and runs it in one step, which is all the QueryOperators +# methods below need to do. +function _scalar(source::Queryable, op::Symbol, f, args...) + return execute_scalar(QueryableScalar(source, op, f, args, source.getiterator)) +end + +# --- The terminal operators --- + +QueryOperators.count(source::Queryable) = + _scalar(source, :count, QueryOperators.count) + +QueryOperators.count(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :count, QueryOperators.count, f, f_expr) + +QueryOperators.any(source::Queryable) = + _scalar(source, :any, QueryOperators.any) + +QueryOperators.any(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :any, QueryOperators.any, f, f_expr) + +QueryOperators.all(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :all, QueryOperators.all, f, f_expr) + +QueryOperators.contains(source::Queryable, value) = + _scalar(source, :contains, QueryOperators.contains, value) + +QueryOperators.min_by(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :min_by, QueryOperators.min_by, f, f_expr) + +QueryOperators.max_by(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :max_by, QueryOperators.max_by, f, f_expr) + +QueryOperators.aggregate(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :aggregate, QueryOperators.aggregate, f, f_expr) + +QueryOperators.aggregate(source::Queryable, seed, f::Function, f_expr::Expr) = + _scalar(source, :aggregate, QueryOperators.aggregate, seed, f, f_expr) + +QueryOperators.first(source::Queryable) = + _scalar(source, :first, QueryOperators.first) + +QueryOperators.first(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :first, QueryOperators.first, f, f_expr) + +QueryOperators.last(source::Queryable) = + _scalar(source, :last, QueryOperators.last) + +QueryOperators.last(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :last, QueryOperators.last, f, f_expr) + +QueryOperators.single(source::Queryable) = + _scalar(source, :single, QueryOperators.single) + +QueryOperators.single(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :single, QueryOperators.single, f, f_expr) + +QueryOperators.element_at(source::Queryable, n::Integer) = + _scalar(source, :element_at, QueryOperators.element_at, n) + +# sequence_equal compares two sequences, so the second one is materialized +# alongside the first rather than being pushed down. +function QueryOperators.sequence_equal(a::Queryable, b) + return _scalar(a, :sequence_equal, QueryOperators.sequence_equal, _materialize_operand(b)) +end + +_materialize_operand(x::Queryable) = _materialize(x) +_materialize_operand(x) = QueryOperators.query(x) diff --git a/src/queryable/queryable_setops.jl b/src/queryable/queryable_setops.jl new file mode 100644 index 0000000..8a4d5e7 --- /dev/null +++ b/src/queryable/queryable_setops.jl @@ -0,0 +1,47 @@ +struct QueryableConcat <: QueryableBinary + outer + inner + getiterator +end + +struct QueryableUnion <: QueryableBinary + outer + inner + f_func + f_expr + getiterator +end + +struct QueryableExcept <: QueryableBinary + outer + inner + f_func + f_expr + getiterator +end + +struct QueryableIntersect <: QueryableBinary + outer + inner + f_func + f_expr + getiterator +end + +function QueryOperators.concat(outer::Queryable, inner) + return QueryableConcat(outer, inner, outer.getiterator) +end + +# The plain and `_by` forms share a node: the plain form records a `nothing` +# key selector, which a backend reads as "compare whole elements". +for (op, op_by, node) in ((:union, :union_by, :QueryableUnion), + (:except, :except_by, :QueryableExcept), + (:intersect, :intersect_by, :QueryableIntersect)) + @eval function QueryOperators.$op(outer::Queryable, inner) + return $node(outer, inner, nothing, nothing, outer.getiterator) + end + + @eval function QueryOperators.$op_by(outer::Queryable, inner, f::Function, f_expr::Expr) + return $node(outer, inner, f, f_expr, outer.getiterator) + end +end diff --git a/src/queryable/queryable_typefiltering.jl b/src/queryable/queryable_typefiltering.jl new file mode 100644 index 0000000..289ad68 --- /dev/null +++ b/src/queryable/queryable_typefiltering.jl @@ -0,0 +1,19 @@ +struct QueryableOfType <: Queryable + source + T::Type + getiterator +end + +struct QueryableCast <: Queryable + source + T::Type + getiterator +end + +function QueryOperators.of_type(source::Queryable, ::Type{T}) where {T} + return QueryableOfType(source, T, source.getiterator) +end + +function QueryOperators.cast(source::Queryable, ::Type{T}) where {T} + return QueryableCast(source, T, source.getiterator) +end diff --git a/test/runtests.jl b/test/runtests.jl index c3554b2..5367d73 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using TestItemRunner include("test_queryablebackend.jl") +include("test_new_operators.jl") @run_package_tests diff --git a/test/test_new_operators.jl b/test/test_new_operators.jl new file mode 100644 index 0000000..1ae6894 --- /dev/null +++ b/test/test_new_operators.jl @@ -0,0 +1,209 @@ +@testsnippet ExprCompare begin + # A quoted lambda carries a LineNumberNode pointing at wherever it was + # written, so two structurally identical selectors are never `==`. Compare + # them with the line information removed. + strip_lines(x) = x + strip_lines(e::Expr) = Expr(e.head, Any[strip_lines(a) for a in e.args if !(a isa LineNumberNode)]...) + + same_expr(a, b) = strip_lines(a) == strip_lines(b) +end + +@testitem "outer joins build binary nodes" setup=[ExampleBackend] begin + other = [(a = 1, c = "x")] + + for (op, node) in ((QueryOperators.left_join, QueryableBackend.QueryableLeftJoin), + (QueryOperators.right_join, QueryableBackend.QueryableRightJoin), + (QueryOperators.full_join, QueryableBackend.QueryableFullJoin)) + q = op(QueryOperators.query(source), other, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (b = i.b, c = j.c), :((i, j) -> (b = i.b, c = j.c))) + + @test q isa node + @test q isa QueryableBackend.QueryableBinary + @test q.inner === other + @test QueryableBackend.get_source(q) isa QueryableBackend.QueryableSource + end +end + +@testitem "the tree walk follows the outer side of a binary node" setup=[ExampleBackend] begin + q = QueryOperators.left_join( + QueryOperators.query(source), [(a = 1, c = "x")], + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (b = i.b, c = j.c), :((i, j) -> (b = i.b, c = j.c))) + + q = QueryOperators.take(q, 1) + + nodes = QueryableBackend.walk_tree(q) + + @test length(nodes) == 3 + @test nodes[1] isa QueryableBackend.QueryableSource + @test nodes[2] isa QueryableBackend.QueryableLeftJoin + @test nodes[3] isa QueryableBackend.QueryableTake +end + +@testitem "set operators build nodes, with nothing marking the plain form" setup=[ExampleBackend, ExprCompare] begin + other = [(a = 3, b = 3)] + + concat = QueryOperators.concat(QueryOperators.query(source), other) + @test concat isa QueryableBackend.QueryableConcat + @test concat isa QueryableBackend.QueryableBinary + + for (plain, by, node) in ((QueryOperators.union, QueryOperators.union_by, QueryableBackend.QueryableUnion), + (QueryOperators.except, QueryOperators.except_by, QueryableBackend.QueryableExcept), + (QueryOperators.intersect, QueryOperators.intersect_by, QueryableBackend.QueryableIntersect)) + q = plain(QueryOperators.query(source), other) + @test q isa node + @test q.f_expr === nothing + + q_by = by(QueryOperators.query(source), other, i -> i.a, :(i -> i.a)) + @test q_by isa node + @test same_expr(q_by.f_expr, :(i -> i.a)) + end +end + +@testitem "partitioning operators build nodes" setup=[ExampleBackend, ExprCompare] begin + q = QueryOperators.take_while(QueryOperators.query(source), i -> i.a < 2, :(i -> i.a < 2)) + @test q isa QueryableBackend.QueryableTakeWhile + @test same_expr(q.f_expr, :(i -> i.a < 2)) + + q = QueryOperators.drop_while(QueryOperators.query(source), i -> i.a < 2, :(i -> i.a < 2)) + @test q isa QueryableBackend.QueryableDropWhile + + q = QueryOperators.take_last(QueryOperators.query(source), 3) + @test q isa QueryableBackend.QueryableTakeLast + @test q.n == 3 + + q = QueryOperators.drop_last(QueryOperators.query(source), 2) + @test q isa QueryableBackend.QueryableDropLast + @test q.n == 2 +end + +@testitem "order reuses QueryableOrderBy with an identity selector" setup=[ExampleBackend, ExprCompare] begin + q = QueryOperators.order(QueryOperators.query(source)) + + @test q isa QueryableBackend.QueryableOrderBy + @test q.descending == false + @test same_expr(q.keySelector_expr, :(i -> i)) + + q = QueryOperators.order_descending(QueryOperators.query(source)) + @test q isa QueryableBackend.QueryableOrderBy + @test q.descending == true + @test same_expr(q.keySelector_expr, :(i -> i)) +end + +@testitem "thenby can still follow order" setup=[ExampleBackend] begin + q = QueryOperators.thenby( + QueryOperators.order(QueryOperators.query(source)), + i -> i.b, :(i -> i.b)) + + @test q isa QueryableBackend.QueryableThenBy + @test q.source isa QueryableBackend.QueryableOrderBy +end + +@testitem "ordering and row-position operators build nodes" setup=[ExampleBackend] begin + @test QueryOperators.reverse(QueryOperators.query(source)) isa QueryableBackend.QueryableReverse + @test QueryOperators.index(QueryOperators.query(source)) isa QueryableBackend.QueryableIndex + + q = QueryOperators.shuffle(QueryOperators.query(source)) + @test q isa QueryableBackend.QueryableShuffle + @test q.rng === nothing + + q = QueryOperators.shuffle(QueryOperators.query(source), 42) + @test q isa QueryableBackend.QueryableShuffle + @test q.rng == 42 +end + +@testitem "combining operators build nodes" setup=[ExampleBackend, ExprCompare] begin + q = QueryOperators.append(QueryOperators.query(source), (a = 9, b = 9)) + @test q isa QueryableBackend.QueryableAppend + @test q.element == (a = 9, b = 9) + + q = QueryOperators.prepend(QueryOperators.query(source), (a = 0, b = 0)) + @test q isa QueryableBackend.QueryablePrepend + + q = QueryOperators.zip(QueryOperators.query(source), [1, 2]) + @test q isa QueryableBackend.QueryableZip + @test q isa QueryableBackend.QueryableBinary + @test q.resultSelector_expr === nothing + + q = QueryOperators.zip(QueryOperators.query(source), [1, 2], (x, y) -> (x, y), :((x, y) -> (x, y))) + @test same_expr(q.resultSelector_expr, :((x, y) -> (x, y))) +end + +@testitem "keyed aggregation operators build nodes" setup=[ExampleBackend] begin + q = QueryOperators.count_by(QueryOperators.query(source), i -> i.a, :(i -> i.a)) + @test q isa QueryableBackend.QueryableCountBy + + q = QueryOperators.aggregate_by(QueryOperators.query(source), i -> i.a, :(i -> i.a), 0, (acc, cur) -> acc + cur.b) + @test q isa QueryableBackend.QueryableAggregateBy + @test q.seed == 0 + + q = QueryOperators.chunk(QueryOperators.query(source), 2) + @test q isa QueryableBackend.QueryableChunk + @test q.n == 2 +end + +@testitem "type filtering operators build nodes" setup=[ExampleBackend] begin + q = QueryOperators.of_type(QueryOperators.query(source), NamedTuple) + @test q isa QueryableBackend.QueryableOfType + @test q.T == NamedTuple + + q = QueryOperators.cast(QueryOperators.query(source), Any) + @test q isa QueryableBackend.QueryableCast + @test q.T == Any +end + +@testitem "terminal operators fall back to the in-memory implementation" setup=[ExampleBackend] begin + q = QueryOperators.query(source) + + # The example backend's getiterator yields [(a=1,b=1), (a=2,b=2)]. + @test QueryOperators.count(q) == 2 + @test QueryOperators.count(q, i -> i.a > 1, :(i -> i.a > 1)) == 1 + @test QueryOperators.any(q) == true + @test QueryOperators.any(q, i -> i.a > 5, :(i -> i.a > 5)) == false + @test QueryOperators.all(q, i -> i.a > 0, :(i -> i.a > 0)) == true + @test QueryOperators.contains(q, (a = 1, b = 1)) == true + @test QueryOperators.first(q) == (a = 1, b = 1) + @test QueryOperators.last(q) == (a = 2, b = 2) + @test QueryOperators.element_at(q, 2) == (a = 2, b = 2) + @test QueryOperators.min_by(q, i -> i.a, :(i -> i.a)) == (a = 1, b = 1) + @test QueryOperators.max_by(q, i -> i.a, :(i -> i.a)) == (a = 2, b = 2) + @test QueryOperators.single(q, i -> i.a == 2, :(i -> i.a == 2)) == (a = 2, b = 2) + @test QueryOperators.aggregate(q, 0, (acc, cur) -> acc + cur.a, :((acc, cur) -> acc + cur.a)) == 3 + @test QueryOperators.sequence_equal(q, [(a = 1, b = 1), (a = 2, b = 2)]) == true + @test QueryOperators.sequence_equal(q, [(a = 1, b = 1)]) == false +end + +@testitem "count on a Queryable used to have no method at all" setup=[ExampleBackend] begin + # Regression: before QueryableScalar there was no QueryOperators.count + # method taking a Queryable, so a query ending in @count() failed. + @test hasmethod(QueryOperators.count, Tuple{QueryableBackend.Queryable}) + @test QueryOperators.count(QueryOperators.query(source)) == 2 +end + +@testitem "the query plan describes the new nodes" setup=[ExampleBackend] begin + q = QueryOperators.index( + QueryOperators.take_last( + QueryOperators.reverse(QueryOperators.query(source)), 2)) + + plan = QueryableBackend.queryplan(q) + descriptions = [QueryableBackend.describe_node(n) for n in plan.nodes] + + @test descriptions == ["Source", "Reverse", "TakeLast: 2", "Index"] + + # None of the new nodes should be falling through to the type-name fallback. + @test !any(d -> startswith(d, "QueryableBackend."), descriptions) +end + +@testitem "the query plan distinguishes the plain and _by set operators" setup=[ExampleBackend] begin + plain = QueryableBackend.describe_node( + QueryOperators.except(QueryOperators.query(source), [(a = 1, b = 1)])) + by = QueryableBackend.describe_node( + QueryOperators.except_by(QueryOperators.query(source), [(a = 1, b = 1)], i -> i.a, :(i -> i.a))) + + @test plain == "Except" + @test startswith(by, "ExceptBy:") + @test occursin("i.a", by) +end From b4141902ae15775ddae6559f6d9b58f6c9b35026 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:47:21 -0700 Subject: [PATCH 3/4] Name the terminal-operator fallback so backends can call it A backend that translates only some terminal operators needs a way to hand the rest back. Splitting _execute_scalar_fallback out of the QueryableSource method gives it one, so its coverage can stay partial without affecting correctness. Co-Authored-By: Claude Opus 5 --- src/queryable/queryable_scalar.jl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/queryable/queryable_scalar.jl b/src/queryable/queryable_scalar.jl index 6f80df3..e8188e4 100644 --- a/src/queryable/queryable_scalar.jl +++ b/src/queryable/queryable_scalar.jl @@ -28,9 +28,16 @@ The fallback materialises the source and delegates to the in-memory """ execute_scalar(q::QueryableScalar) = _execute_scalar(get_source(q.source), q) -function _execute_scalar(::QueryableSource, q::QueryableScalar) - return q.f(_materialize(q.source), q.args...) -end +_execute_scalar(::QueryableSource, q::QueryableScalar) = _execute_scalar_fallback(q) + +""" + _execute_scalar_fallback(q::QueryableScalar) + +Materialize the source and run the in-memory implementation. A backend that +translates only some terminal operators calls this for the rest, so that +correctness never depends on its coverage being complete. +""" +_execute_scalar_fallback(q::QueryableScalar) = q.f(_materialize(q.source), q.args...) _materialize(source) = QueryOperators.query(IteratorInterfaceExtensions.getiterator(source)) From 483ad6948f1d8f843ebb6ec09d1f5172ac8375f4 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Thu, 17 Sep 2026 21:50:03 -0700 Subject: [PATCH 4/4] Require Julia 1.10, as QueryOperators 1.2 does The compat bound on QueryOperators moved to 1.2, which requires Julia 1.10. The julia bound here was still "1", and the CI workflow generates its matrix from it, so every job from Julia 1.0 to 1.9 spawned and then failed to resolve a QueryOperators that could satisfy both bounds. Co-Authored-By: Claude Opus 5 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9ad87e6..62134ff 100644 --- a/Project.toml +++ b/Project.toml @@ -13,7 +13,7 @@ TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -julia = "1" +julia = "1.10" QueryOperators = "1.2" IteratorInterfaceExtensions = "1" TableTraits = "1"