diff --git a/Project.toml b/Project.toml index ff953bd..62134ff 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "QueryableBackend" uuid = "0898d9ac-042b-5d31-8dcc-959fb3365f19" -version = "1.0.1-DEV" +version = "1.1.0-DEV" [deps] QueryOperators = "2aef5ad7-51ca-5a8f-8e88-e75cf067b44b" @@ -13,8 +13,8 @@ TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [compat] -julia = "1" -QueryOperators = "0.8, 0.9, 1" +julia = "1.10" +QueryOperators = "1.2" IteratorInterfaceExtensions = "1" TableTraits = "1" diff --git a/src/QueryableBackend.jl b/src/QueryableBackend.jl index 08af8f0..cc61851 100644 --- a/src/QueryableBackend.jl +++ b/src/QueryableBackend.jl @@ -10,13 +10,21 @@ include("queryable/queryable_orderby.jl") include("queryable/queryable_thenby.jl") include("queryable/queryable_join.jl") include("queryable/queryable_groupjoin.jl") +include("queryable/queryable_outerjoin.jl") include("queryable/queryable_mapmany.jl") include("queryable/queryable_take.jl") include("queryable/queryable_drop.jl") include("queryable/queryable_unique.jl") +include("queryable/queryable_setops.jl") +include("queryable/queryable_partition.jl") +include("queryable/queryable_ordering.jl") +include("queryable/queryable_combining.jl") +include("queryable/queryable_keyedaggregation.jl") +include("queryable/queryable_typefiltering.jl") include("source_queryable.jl") include("query_tree.jl") +include("queryable/queryable_scalar.jl") include("query_plan.jl") end diff --git a/src/query_plan.jl b/src/query_plan.jl index 655ac42..6f98777 100644 --- a/src/query_plan.jl +++ b/src/query_plan.jl @@ -71,6 +71,65 @@ function describe_node(q::QueryableMapMany) return "MapMany: $(string(q.collectionSelector_expr)) => $(string(q.resultSelector_expr))" end +function describe_node(q::QueryableLeftJoin) + return "LeftJoin: $(string(q.outerKeySelector_expr)) = $(string(q.innerKeySelector_expr)) => $(string(q.resultSelector_expr))" +end + +function describe_node(q::QueryableRightJoin) + return "RightJoin: $(string(q.outerKeySelector_expr)) = $(string(q.innerKeySelector_expr)) => $(string(q.resultSelector_expr))" +end + +function describe_node(q::QueryableFullJoin) + return "FullJoin: $(string(q.outerKeySelector_expr)) = $(string(q.innerKeySelector_expr)) => $(string(q.resultSelector_expr))" +end + +describe_node(::QueryableConcat) = "Concat" + +describe_node(q::QueryableUnion) = _describe_setop("Union", q) + +describe_node(q::QueryableExcept) = _describe_setop("Except", q) + +describe_node(q::QueryableIntersect) = _describe_setop("Intersect", q) + +# A `nothing` key selector is the plain form, which compares whole elements. +function _describe_setop(name, q) + return q.f_expr === nothing ? name : "$(name)By: $(string(q.f_expr))" +end + +describe_node(q::QueryableTakeWhile) = "TakeWhile: $(string(q.f_expr))" + +describe_node(q::QueryableDropWhile) = "DropWhile: $(string(q.f_expr))" + +describe_node(q::QueryableTakeLast) = "TakeLast: $(q.n)" + +describe_node(q::QueryableDropLast) = "DropLast: $(q.n)" + +describe_node(::QueryableReverse) = "Reverse" + +describe_node(::QueryableShuffle) = "Shuffle" + +describe_node(::QueryableIndex) = "Index" + +describe_node(q::QueryableAppend) = "Append: $(repr(q.element))" + +describe_node(q::QueryablePrepend) = "Prepend: $(repr(q.element))" + +function describe_node(q::QueryableZip) + return q.resultSelector_expr === nothing ? "Zip" : "Zip: $(string(q.resultSelector_expr))" +end + +describe_node(q::QueryableCountBy) = "CountBy: $(string(q.f_expr))" + +describe_node(q::QueryableAggregateBy) = "AggregateBy: $(string(q.f_expr))" + +describe_node(q::QueryableChunk) = "Chunk: $(q.n)" + +describe_node(q::QueryableOfType) = "OfType: $(q.T)" + +describe_node(q::QueryableCast) = "Cast: $(q.T)" + +describe_node(q::QueryableScalar) = "Scalar: $(q.op)" + # Fallback for unknown/extension types describe_node(q::Queryable) = string(typeof(q)) diff --git a/src/query_tree.jl b/src/query_tree.jl index 703764e..8d6cb97 100644 --- a/src/query_tree.jl +++ b/src/query_tree.jl @@ -1,17 +1,14 @@ """ get_source(q::Queryable) -Follow the `.source` (or `.outer`) chain to the root `QueryableSource`. +Follow the `.source` (or, for two-input nodes, `.outer`) chain to the root +`QueryableSource`. """ function get_source(q::QueryableSource) return q end -function get_source(q::QueryableJoin) - return get_source(q.outer) -end - -function get_source(q::QueryableGroupJoin) +function get_source(q::QueryableBinary) return get_source(q.outer) end @@ -34,12 +31,7 @@ function _collect_nodes!(nodes, q::QueryableSource) push!(nodes, q) end -function _collect_nodes!(nodes, q::QueryableJoin) - push!(nodes, q) - _collect_nodes!(nodes, q.outer) -end - -function _collect_nodes!(nodes, q::QueryableGroupJoin) +function _collect_nodes!(nodes, q::QueryableBinary) push!(nodes, q) _collect_nodes!(nodes, q.outer) end diff --git a/src/queryable/queryable.jl b/src/queryable/queryable.jl index b13c294..82312c9 100644 --- a/src/queryable/queryable.jl +++ b/src/queryable/queryable.jl @@ -1,5 +1,15 @@ abstract type Queryable end +""" + QueryableBinary <: Queryable + +Nodes that combine two sources — joins, set operations, `zip`. They carry +`outer` and `inner` fields, and the tree walk follows `outer`, so the query +plan reads as a chain hanging off the primary input with the secondary input +attached at the node. +""" +abstract type QueryableBinary <: Queryable end + QueryOperators.query(x::Queryable) = x IteratorInterfaceExtensions.isiterable(x::Queryable) = true diff --git a/src/queryable/queryable_combining.jl b/src/queryable/queryable_combining.jl new file mode 100644 index 0000000..a8e7aa8 --- /dev/null +++ b/src/queryable/queryable_combining.jl @@ -0,0 +1,37 @@ +struct QueryableAppend <: Queryable + source + element + getiterator +end + +struct QueryablePrepend <: Queryable + source + element + getiterator +end + +struct QueryableZip <: QueryableBinary + outer + inner + resultSelector_func + resultSelector_expr + getiterator +end + +function QueryOperators.append(source::Queryable, element) + return QueryableAppend(source, element, source.getiterator) +end + +function QueryOperators.prepend(source::Queryable, element) + return QueryablePrepend(source, element, source.getiterator) +end + +# As with the set operators, a `nothing` result selector means the default: +# pair the two elements into a tuple. +function QueryOperators.zip(outer::Queryable, inner) + return QueryableZip(outer, inner, nothing, nothing, outer.getiterator) +end + +function QueryOperators.zip(outer::Queryable, inner, f_resultSelector::Function, resultSelector::Expr) + return QueryableZip(outer, inner, f_resultSelector, resultSelector, outer.getiterator) +end diff --git a/src/queryable/queryable_groupjoin.jl b/src/queryable/queryable_groupjoin.jl index 6dfd0f1..33baa49 100644 --- a/src/queryable/queryable_groupjoin.jl +++ b/src/queryable/queryable_groupjoin.jl @@ -1,4 +1,4 @@ -struct QueryableGroupJoin <: Queryable +struct QueryableGroupJoin <: QueryableBinary outer inner outerKeySelector_func diff --git a/src/queryable/queryable_join.jl b/src/queryable/queryable_join.jl index 4dde26e..dab1a61 100644 --- a/src/queryable/queryable_join.jl +++ b/src/queryable/queryable_join.jl @@ -1,4 +1,4 @@ -struct QueryableJoin <: Queryable +struct QueryableJoin <: QueryableBinary outer inner outerKeySelector_func diff --git a/src/queryable/queryable_keyedaggregation.jl b/src/queryable/queryable_keyedaggregation.jl new file mode 100644 index 0000000..75b63fc --- /dev/null +++ b/src/queryable/queryable_keyedaggregation.jl @@ -0,0 +1,33 @@ +struct QueryableCountBy <: Queryable + source + f_func + f_expr + getiterator +end + +struct QueryableAggregateBy <: Queryable + source + f_func + f_expr + seed + accumulator_func + getiterator +end + +struct QueryableChunk <: Queryable + source + n::Int + getiterator +end + +function QueryOperators.count_by(source::Queryable, f::Function, f_expr::Expr) + return QueryableCountBy(source, f, f_expr, source.getiterator) +end + +function QueryOperators.aggregate_by(source::Queryable, f::Function, f_expr::Expr, seed, accumulator::Function) + return QueryableAggregateBy(source, f, f_expr, seed, accumulator, source.getiterator) +end + +function QueryOperators.chunk(source::Queryable, n::Integer) + return QueryableChunk(source, Int(n), source.getiterator) +end diff --git a/src/queryable/queryable_ordering.jl b/src/queryable/queryable_ordering.jl new file mode 100644 index 0000000..adbfba2 --- /dev/null +++ b/src/queryable/queryable_ordering.jl @@ -0,0 +1,43 @@ +struct QueryableReverse <: Queryable + source + getiterator +end + +struct QueryableShuffle <: Queryable + source + rng + getiterator +end + +struct QueryableIndex <: Queryable + source + getiterator +end + +# `order` and `order_descending` sort by the elements themselves, which is an +# orderby with an identity key selector. They reuse QueryableOrderBy rather +# than adding a node type, so `thenby` can still follow them and a backend can +# recognise the identity selector to sort by every column. +function QueryOperators.order(source::Queryable) + return QueryOperators.orderby(source, identity, :(i -> i)) +end + +function QueryOperators.order_descending(source::Queryable) + return QueryOperators.orderby_descending(source, identity, :(i -> i)) +end + +function QueryOperators.reverse(source::Queryable) + return QueryableReverse(source, source.getiterator) +end + +function QueryOperators.shuffle(source::Queryable) + return QueryableShuffle(source, nothing, source.getiterator) +end + +function QueryOperators.shuffle(source::Queryable, rng) + return QueryableShuffle(source, rng, source.getiterator) +end + +function QueryOperators.index(source::Queryable) + return QueryableIndex(source, source.getiterator) +end diff --git a/src/queryable/queryable_outerjoin.jl b/src/queryable/queryable_outerjoin.jl new file mode 100644 index 0000000..a3c271e --- /dev/null +++ b/src/queryable/queryable_outerjoin.jl @@ -0,0 +1,43 @@ +struct QueryableLeftJoin <: QueryableBinary + outer + inner + outerKeySelector_func + outerKeySelector_expr + innerKeySelector_func + innerKeySelector_expr + resultSelector_func + resultSelector_expr + getiterator +end + +struct QueryableRightJoin <: QueryableBinary + outer + inner + outerKeySelector_func + outerKeySelector_expr + innerKeySelector_func + innerKeySelector_expr + resultSelector_func + resultSelector_expr + getiterator +end + +struct QueryableFullJoin <: QueryableBinary + outer + inner + outerKeySelector_func + outerKeySelector_expr + innerKeySelector_func + innerKeySelector_expr + resultSelector_func + resultSelector_expr + getiterator +end + +for (op, node) in ((:left_join, :QueryableLeftJoin), + (:right_join, :QueryableRightJoin), + (:full_join, :QueryableFullJoin)) + @eval function QueryOperators.$op(outer::Queryable, inner, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + return $node(outer, inner, f_outerKeySelector, outerKeySelector, f_innerKeySelector, innerKeySelector, f_resultSelector, resultSelector, outer.getiterator) + end +end diff --git a/src/queryable/queryable_partition.jl b/src/queryable/queryable_partition.jl new file mode 100644 index 0000000..b2e73de --- /dev/null +++ b/src/queryable/queryable_partition.jl @@ -0,0 +1,41 @@ +struct QueryableTakeWhile <: Queryable + source + f_func + f_expr + getiterator +end + +struct QueryableDropWhile <: Queryable + source + f_func + f_expr + getiterator +end + +struct QueryableTakeLast <: Queryable + source + n::Int + getiterator +end + +struct QueryableDropLast <: Queryable + source + n::Int + getiterator +end + +function QueryOperators.take_while(source::Queryable, f::Function, f_expr::Expr) + return QueryableTakeWhile(source, f, f_expr, source.getiterator) +end + +function QueryOperators.drop_while(source::Queryable, f::Function, f_expr::Expr) + return QueryableDropWhile(source, f, f_expr, source.getiterator) +end + +function QueryOperators.take_last(source::Queryable, n::Integer) + return QueryableTakeLast(source, Int(n), source.getiterator) +end + +function QueryOperators.drop_last(source::Queryable, n::Integer) + return QueryableDropLast(source, Int(n), source.getiterator) +end diff --git a/src/queryable/queryable_scalar.jl b/src/queryable/queryable_scalar.jl new file mode 100644 index 0000000..e8188e4 --- /dev/null +++ b/src/queryable/queryable_scalar.jl @@ -0,0 +1,110 @@ +""" + QueryableScalar <: Queryable + +A terminal operator — one that returns a value rather than another sequence. + +Unlike every other node this is never left sitting in a tree: the +`QueryOperators` method that builds it immediately hands it to +[`execute_scalar`](@ref). It exists so that a backend has something to inspect +when it wants to push the operator down instead of materialising the source. +""" +struct QueryableScalar <: Queryable + source + op::Symbol + f + args::Tuple + getiterator +end + +""" + execute_scalar(q::QueryableScalar) + +Produce the value of a terminal operator. + +Dispatches on the type of the root source, so a backend can specialise +`_execute_scalar` for its own source type and translate the operator instead. +The fallback materialises the source and delegates to the in-memory +`QueryOperators` implementation, which every backend gets for free. +""" +execute_scalar(q::QueryableScalar) = _execute_scalar(get_source(q.source), q) + +_execute_scalar(::QueryableSource, q::QueryableScalar) = _execute_scalar_fallback(q) + +""" + _execute_scalar_fallback(q::QueryableScalar) + +Materialize the source and run the in-memory implementation. A backend that +translates only some terminal operators calls this for the rest, so that +correctness never depends on its coverage being complete. +""" +_execute_scalar_fallback(q::QueryableScalar) = q.f(_materialize(q.source), q.args...) + +_materialize(source) = QueryOperators.query(IteratorInterfaceExtensions.getiterator(source)) + +# Builds the node and runs it in one step, which is all the QueryOperators +# methods below need to do. +function _scalar(source::Queryable, op::Symbol, f, args...) + return execute_scalar(QueryableScalar(source, op, f, args, source.getiterator)) +end + +# --- The terminal operators --- + +QueryOperators.count(source::Queryable) = + _scalar(source, :count, QueryOperators.count) + +QueryOperators.count(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :count, QueryOperators.count, f, f_expr) + +QueryOperators.any(source::Queryable) = + _scalar(source, :any, QueryOperators.any) + +QueryOperators.any(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :any, QueryOperators.any, f, f_expr) + +QueryOperators.all(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :all, QueryOperators.all, f, f_expr) + +QueryOperators.contains(source::Queryable, value) = + _scalar(source, :contains, QueryOperators.contains, value) + +QueryOperators.min_by(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :min_by, QueryOperators.min_by, f, f_expr) + +QueryOperators.max_by(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :max_by, QueryOperators.max_by, f, f_expr) + +QueryOperators.aggregate(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :aggregate, QueryOperators.aggregate, f, f_expr) + +QueryOperators.aggregate(source::Queryable, seed, f::Function, f_expr::Expr) = + _scalar(source, :aggregate, QueryOperators.aggregate, seed, f, f_expr) + +QueryOperators.first(source::Queryable) = + _scalar(source, :first, QueryOperators.first) + +QueryOperators.first(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :first, QueryOperators.first, f, f_expr) + +QueryOperators.last(source::Queryable) = + _scalar(source, :last, QueryOperators.last) + +QueryOperators.last(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :last, QueryOperators.last, f, f_expr) + +QueryOperators.single(source::Queryable) = + _scalar(source, :single, QueryOperators.single) + +QueryOperators.single(source::Queryable, f::Function, f_expr::Expr) = + _scalar(source, :single, QueryOperators.single, f, f_expr) + +QueryOperators.element_at(source::Queryable, n::Integer) = + _scalar(source, :element_at, QueryOperators.element_at, n) + +# sequence_equal compares two sequences, so the second one is materialized +# alongside the first rather than being pushed down. +function QueryOperators.sequence_equal(a::Queryable, b) + return _scalar(a, :sequence_equal, QueryOperators.sequence_equal, _materialize_operand(b)) +end + +_materialize_operand(x::Queryable) = _materialize(x) +_materialize_operand(x) = QueryOperators.query(x) diff --git a/src/queryable/queryable_setops.jl b/src/queryable/queryable_setops.jl new file mode 100644 index 0000000..8a4d5e7 --- /dev/null +++ b/src/queryable/queryable_setops.jl @@ -0,0 +1,47 @@ +struct QueryableConcat <: QueryableBinary + outer + inner + getiterator +end + +struct QueryableUnion <: QueryableBinary + outer + inner + f_func + f_expr + getiterator +end + +struct QueryableExcept <: QueryableBinary + outer + inner + f_func + f_expr + getiterator +end + +struct QueryableIntersect <: QueryableBinary + outer + inner + f_func + f_expr + getiterator +end + +function QueryOperators.concat(outer::Queryable, inner) + return QueryableConcat(outer, inner, outer.getiterator) +end + +# The plain and `_by` forms share a node: the plain form records a `nothing` +# key selector, which a backend reads as "compare whole elements". +for (op, op_by, node) in ((:union, :union_by, :QueryableUnion), + (:except, :except_by, :QueryableExcept), + (:intersect, :intersect_by, :QueryableIntersect)) + @eval function QueryOperators.$op(outer::Queryable, inner) + return $node(outer, inner, nothing, nothing, outer.getiterator) + end + + @eval function QueryOperators.$op_by(outer::Queryable, inner, f::Function, f_expr::Expr) + return $node(outer, inner, f, f_expr, outer.getiterator) + end +end diff --git a/src/queryable/queryable_typefiltering.jl b/src/queryable/queryable_typefiltering.jl new file mode 100644 index 0000000..289ad68 --- /dev/null +++ b/src/queryable/queryable_typefiltering.jl @@ -0,0 +1,19 @@ +struct QueryableOfType <: Queryable + source + T::Type + getiterator +end + +struct QueryableCast <: Queryable + source + T::Type + getiterator +end + +function QueryOperators.of_type(source::Queryable, ::Type{T}) where {T} + return QueryableOfType(source, T, source.getiterator) +end + +function QueryOperators.cast(source::Queryable, ::Type{T}) where {T} + return QueryableCast(source, T, source.getiterator) +end diff --git a/test/runtests.jl b/test/runtests.jl index c3554b2..5367d73 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,6 @@ using TestItemRunner include("test_queryablebackend.jl") +include("test_new_operators.jl") @run_package_tests diff --git a/test/test_new_operators.jl b/test/test_new_operators.jl new file mode 100644 index 0000000..1ae6894 --- /dev/null +++ b/test/test_new_operators.jl @@ -0,0 +1,209 @@ +@testsnippet ExprCompare begin + # A quoted lambda carries a LineNumberNode pointing at wherever it was + # written, so two structurally identical selectors are never `==`. Compare + # them with the line information removed. + strip_lines(x) = x + strip_lines(e::Expr) = Expr(e.head, Any[strip_lines(a) for a in e.args if !(a isa LineNumberNode)]...) + + same_expr(a, b) = strip_lines(a) == strip_lines(b) +end + +@testitem "outer joins build binary nodes" setup=[ExampleBackend] begin + other = [(a = 1, c = "x")] + + for (op, node) in ((QueryOperators.left_join, QueryableBackend.QueryableLeftJoin), + (QueryOperators.right_join, QueryableBackend.QueryableRightJoin), + (QueryOperators.full_join, QueryableBackend.QueryableFullJoin)) + q = op(QueryOperators.query(source), other, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (b = i.b, c = j.c), :((i, j) -> (b = i.b, c = j.c))) + + @test q isa node + @test q isa QueryableBackend.QueryableBinary + @test q.inner === other + @test QueryableBackend.get_source(q) isa QueryableBackend.QueryableSource + end +end + +@testitem "the tree walk follows the outer side of a binary node" setup=[ExampleBackend] begin + q = QueryOperators.left_join( + QueryOperators.query(source), [(a = 1, c = "x")], + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (b = i.b, c = j.c), :((i, j) -> (b = i.b, c = j.c))) + + q = QueryOperators.take(q, 1) + + nodes = QueryableBackend.walk_tree(q) + + @test length(nodes) == 3 + @test nodes[1] isa QueryableBackend.QueryableSource + @test nodes[2] isa QueryableBackend.QueryableLeftJoin + @test nodes[3] isa QueryableBackend.QueryableTake +end + +@testitem "set operators build nodes, with nothing marking the plain form" setup=[ExampleBackend, ExprCompare] begin + other = [(a = 3, b = 3)] + + concat = QueryOperators.concat(QueryOperators.query(source), other) + @test concat isa QueryableBackend.QueryableConcat + @test concat isa QueryableBackend.QueryableBinary + + for (plain, by, node) in ((QueryOperators.union, QueryOperators.union_by, QueryableBackend.QueryableUnion), + (QueryOperators.except, QueryOperators.except_by, QueryableBackend.QueryableExcept), + (QueryOperators.intersect, QueryOperators.intersect_by, QueryableBackend.QueryableIntersect)) + q = plain(QueryOperators.query(source), other) + @test q isa node + @test q.f_expr === nothing + + q_by = by(QueryOperators.query(source), other, i -> i.a, :(i -> i.a)) + @test q_by isa node + @test same_expr(q_by.f_expr, :(i -> i.a)) + end +end + +@testitem "partitioning operators build nodes" setup=[ExampleBackend, ExprCompare] begin + q = QueryOperators.take_while(QueryOperators.query(source), i -> i.a < 2, :(i -> i.a < 2)) + @test q isa QueryableBackend.QueryableTakeWhile + @test same_expr(q.f_expr, :(i -> i.a < 2)) + + q = QueryOperators.drop_while(QueryOperators.query(source), i -> i.a < 2, :(i -> i.a < 2)) + @test q isa QueryableBackend.QueryableDropWhile + + q = QueryOperators.take_last(QueryOperators.query(source), 3) + @test q isa QueryableBackend.QueryableTakeLast + @test q.n == 3 + + q = QueryOperators.drop_last(QueryOperators.query(source), 2) + @test q isa QueryableBackend.QueryableDropLast + @test q.n == 2 +end + +@testitem "order reuses QueryableOrderBy with an identity selector" setup=[ExampleBackend, ExprCompare] begin + q = QueryOperators.order(QueryOperators.query(source)) + + @test q isa QueryableBackend.QueryableOrderBy + @test q.descending == false + @test same_expr(q.keySelector_expr, :(i -> i)) + + q = QueryOperators.order_descending(QueryOperators.query(source)) + @test q isa QueryableBackend.QueryableOrderBy + @test q.descending == true + @test same_expr(q.keySelector_expr, :(i -> i)) +end + +@testitem "thenby can still follow order" setup=[ExampleBackend] begin + q = QueryOperators.thenby( + QueryOperators.order(QueryOperators.query(source)), + i -> i.b, :(i -> i.b)) + + @test q isa QueryableBackend.QueryableThenBy + @test q.source isa QueryableBackend.QueryableOrderBy +end + +@testitem "ordering and row-position operators build nodes" setup=[ExampleBackend] begin + @test QueryOperators.reverse(QueryOperators.query(source)) isa QueryableBackend.QueryableReverse + @test QueryOperators.index(QueryOperators.query(source)) isa QueryableBackend.QueryableIndex + + q = QueryOperators.shuffle(QueryOperators.query(source)) + @test q isa QueryableBackend.QueryableShuffle + @test q.rng === nothing + + q = QueryOperators.shuffle(QueryOperators.query(source), 42) + @test q isa QueryableBackend.QueryableShuffle + @test q.rng == 42 +end + +@testitem "combining operators build nodes" setup=[ExampleBackend, ExprCompare] begin + q = QueryOperators.append(QueryOperators.query(source), (a = 9, b = 9)) + @test q isa QueryableBackend.QueryableAppend + @test q.element == (a = 9, b = 9) + + q = QueryOperators.prepend(QueryOperators.query(source), (a = 0, b = 0)) + @test q isa QueryableBackend.QueryablePrepend + + q = QueryOperators.zip(QueryOperators.query(source), [1, 2]) + @test q isa QueryableBackend.QueryableZip + @test q isa QueryableBackend.QueryableBinary + @test q.resultSelector_expr === nothing + + q = QueryOperators.zip(QueryOperators.query(source), [1, 2], (x, y) -> (x, y), :((x, y) -> (x, y))) + @test same_expr(q.resultSelector_expr, :((x, y) -> (x, y))) +end + +@testitem "keyed aggregation operators build nodes" setup=[ExampleBackend] begin + q = QueryOperators.count_by(QueryOperators.query(source), i -> i.a, :(i -> i.a)) + @test q isa QueryableBackend.QueryableCountBy + + q = QueryOperators.aggregate_by(QueryOperators.query(source), i -> i.a, :(i -> i.a), 0, (acc, cur) -> acc + cur.b) + @test q isa QueryableBackend.QueryableAggregateBy + @test q.seed == 0 + + q = QueryOperators.chunk(QueryOperators.query(source), 2) + @test q isa QueryableBackend.QueryableChunk + @test q.n == 2 +end + +@testitem "type filtering operators build nodes" setup=[ExampleBackend] begin + q = QueryOperators.of_type(QueryOperators.query(source), NamedTuple) + @test q isa QueryableBackend.QueryableOfType + @test q.T == NamedTuple + + q = QueryOperators.cast(QueryOperators.query(source), Any) + @test q isa QueryableBackend.QueryableCast + @test q.T == Any +end + +@testitem "terminal operators fall back to the in-memory implementation" setup=[ExampleBackend] begin + q = QueryOperators.query(source) + + # The example backend's getiterator yields [(a=1,b=1), (a=2,b=2)]. + @test QueryOperators.count(q) == 2 + @test QueryOperators.count(q, i -> i.a > 1, :(i -> i.a > 1)) == 1 + @test QueryOperators.any(q) == true + @test QueryOperators.any(q, i -> i.a > 5, :(i -> i.a > 5)) == false + @test QueryOperators.all(q, i -> i.a > 0, :(i -> i.a > 0)) == true + @test QueryOperators.contains(q, (a = 1, b = 1)) == true + @test QueryOperators.first(q) == (a = 1, b = 1) + @test QueryOperators.last(q) == (a = 2, b = 2) + @test QueryOperators.element_at(q, 2) == (a = 2, b = 2) + @test QueryOperators.min_by(q, i -> i.a, :(i -> i.a)) == (a = 1, b = 1) + @test QueryOperators.max_by(q, i -> i.a, :(i -> i.a)) == (a = 2, b = 2) + @test QueryOperators.single(q, i -> i.a == 2, :(i -> i.a == 2)) == (a = 2, b = 2) + @test QueryOperators.aggregate(q, 0, (acc, cur) -> acc + cur.a, :((acc, cur) -> acc + cur.a)) == 3 + @test QueryOperators.sequence_equal(q, [(a = 1, b = 1), (a = 2, b = 2)]) == true + @test QueryOperators.sequence_equal(q, [(a = 1, b = 1)]) == false +end + +@testitem "count on a Queryable used to have no method at all" setup=[ExampleBackend] begin + # Regression: before QueryableScalar there was no QueryOperators.count + # method taking a Queryable, so a query ending in @count() failed. + @test hasmethod(QueryOperators.count, Tuple{QueryableBackend.Queryable}) + @test QueryOperators.count(QueryOperators.query(source)) == 2 +end + +@testitem "the query plan describes the new nodes" setup=[ExampleBackend] begin + q = QueryOperators.index( + QueryOperators.take_last( + QueryOperators.reverse(QueryOperators.query(source)), 2)) + + plan = QueryableBackend.queryplan(q) + descriptions = [QueryableBackend.describe_node(n) for n in plan.nodes] + + @test descriptions == ["Source", "Reverse", "TakeLast: 2", "Index"] + + # None of the new nodes should be falling through to the type-name fallback. + @test !any(d -> startswith(d, "QueryableBackend."), descriptions) +end + +@testitem "the query plan distinguishes the plain and _by set operators" setup=[ExampleBackend] begin + plain = QueryableBackend.describe_node( + QueryOperators.except(QueryOperators.query(source), [(a = 1, b = 1)])) + by = QueryableBackend.describe_node( + QueryOperators.except_by(QueryOperators.query(source), [(a = 1, b = 1)], i -> i.a, :(i -> i.a))) + + @test plain == "Except" + @test startswith(by, "ExceptBy:") + @test occursin("i.a", by) +end