From 7c4b028c2f362af0ab07a0c67dd50129140c9786 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:04:27 -0700 Subject: [PATCH 1/9] Add the left_join, right_join and full_join operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports Enumerable.LeftJoin, Enumerable.RightJoin and Enumerable.FullJoin, which .NET 11 added to System.Linq. The unmatched side of an outer join needs an element to pair with. That element is built by the new `_default_value`, a generated runtime counterpart of the existing `_default_value_expr`, so absent values are DataValue and never Missing. Matched elements on a nullable side are converted to the same all-null-capable type, which keeps a single element type across matched and unmatched rows — the same trick EnumerableDefaultIfEmpty already uses. full_join emits every outer row in source order first, then the inner rows whose key never appeared on the outer side. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 3 + src/enumerable/enumerable_defaultifempty.jl | 8 + src/enumerable/enumerable_fulljoin.jl | 80 ++++++++ src/enumerable/enumerable_leftjoin.jl | 85 +++++++++ src/enumerable/enumerable_rightjoin.jl | 63 +++++++ src/operators.jl | 9 + test/runtests.jl | 1 + test/test_enumerable_outerjoins.jl | 191 ++++++++++++++++++++ 8 files changed, 440 insertions(+) create mode 100644 src/enumerable/enumerable_fulljoin.jl create mode 100644 src/enumerable/enumerable_leftjoin.jl create mode 100644 src/enumerable/enumerable_rightjoin.jl create mode 100644 test/test_enumerable_outerjoins.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index 82c2379..d0fd967 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -14,6 +14,9 @@ include("enumerable/enumerable.jl") include("enumerable/enumerable_groupby.jl") include("enumerable/enumerable_join.jl") include("enumerable/enumerable_groupjoin.jl") +include("enumerable/enumerable_leftjoin.jl") +include("enumerable/enumerable_rightjoin.jl") +include("enumerable/enumerable_fulljoin.jl") include("enumerable/enumerable_orderby.jl") include("enumerable/enumerable_map.jl") include("enumerable/enumerable_filter.jl") diff --git a/src/enumerable/enumerable_defaultifempty.jl b/src/enumerable/enumerable_defaultifempty.jl index 9a9c7c4..45d10fa 100644 --- a/src/enumerable/enumerable_defaultifempty.jl +++ b/src/enumerable/enumerable_defaultifempty.jl @@ -13,6 +13,14 @@ function _default_value_expr(::Type{T}) where {T<:NamedTuple} return :( NamedTuple{$(fieldnames(T))}( ($( (_default_value_expr(fieldtype(T,i)) for i in 1:length(fieldnames(T)))... ),)) ) end +# Runtime counterpart of `_default_value_expr`, used by the outer join operators +# to build the all-null element that an unmatched side contributes. Generated so +# that the NamedTuple case is constructed at compile time, exactly as +# `default_if_empty` below does. +@generated function _default_value(::Type{T}) where {T} + return _default_value_expr(T) +end + @generated function default_if_empty(source::S) where {S} T_source = eltype(source) diff --git a/src/enumerable/enumerable_fulljoin.jl b/src/enumerable/enumerable_fulljoin.jl new file mode 100644 index 0000000..8420e32 --- /dev/null +++ b/src/enumerable/enumerable_fulljoin.jl @@ -0,0 +1,80 @@ +struct EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS<:Function,IKS<:Function,RS<:Function} <: Enumerable + outer::SO + inner::SI + outerKeySelector::OKS + innerKeySelector::IKS + resultSelector::RS + default_outer::TOD + default_inner::TID +end + +Base.eltype(::Type{EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS,IKS,RS}}) where {T,TKey,TOD,TID,SO,SI,OKS,IKS,RS} = T + +function full_join(outer::Enumerable, inner::Enumerable, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + TO = eltype(outer) + TI = eltype(inner) + TKeyOuter = Base._return_type(f_outerKeySelector, Tuple{TO,}) + TKeyInner = Base._return_type(f_innerKeySelector, Tuple{TI,}) + + _check_join_key_types("full_join", TKeyOuter, TKeyInner) + + default_outer = _default_value(TO) + default_inner = _default_value(TI) + TOD = typeof(default_outer) + TID = typeof(default_inner) + + T = Base._return_type(f_resultSelector, Tuple{TOD,TID}) + + SO = typeof(outer) + SI = typeof(inner) + OKS = typeof(f_outerKeySelector) + IKS = typeof(f_innerKeySelector) + RS = typeof(f_resultSelector) + + return EnumerableFullJoin{T,TKeyOuter,TOD,TID,SO,SI,OKS,IKS,RS}(outer, inner, f_outerKeySelector, f_innerKeySelector, f_resultSelector, default_outer, default_inner) +end + +function Base.iterate(iter::EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS,IKS,RS}) where {T,TKey,TOD,TID,SO,SI,OKS,IKS,RS} + results = Array{T}(undef, 0) + + inner_dict = _outerjoin_lookup(iter.inner, iter.innerKeySelector, TKey, TID) + + # All outer elements first, in source order: matched pairs where a key + # matches, otherwise the outer element paired with an all-null inner. + matched_keys = Set{TKey}() + for i in iter.outer + outerKey = iter.outerKeySelector(i) + converted_i = convert(TOD, i) + if haskey(inner_dict, outerKey) + push!(matched_keys, outerKey) + for j in inner_dict[outerKey] + push!(results, iter.resultSelector(converted_i, j)) + end + else + push!(results, iter.resultSelector(converted_i, iter.default_inner)) + end + end + + # Then the inner elements whose key never appeared on the outer side, + # in inner source order, paired with an all-null outer. + for (innerKey, elements) in inner_dict + innerKey in matched_keys && continue + for j in elements + push!(results, iter.resultSelector(iter.default_outer, j)) + end + end + + if length(results)==0 + return nothing + end + + return results[1], (results, 2) +end + +function Base.iterate(iter::EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS,IKS,RS}, state) where {T,TKey,TOD,TID,SO,SI,OKS,IKS,RS} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_leftjoin.jl b/src/enumerable/enumerable_leftjoin.jl new file mode 100644 index 0000000..2289ce1 --- /dev/null +++ b/src/enumerable/enumerable_leftjoin.jl @@ -0,0 +1,85 @@ +# Builds the lookup from join key to the list of elements on one side of an +# outer join. Elements are converted to `TD` — the all-null-capable version of +# their own type — so that matched and unmatched rows have a single element +# type, mirroring how `EnumerableDefaultIfEmpty` converts its source. +function _outerjoin_lookup(source, keySelector, ::Type{TKey}, ::Type{TD}) where {TKey,TD} + lookup = OrderedDict{TKey,Array{TD,1}}() + for i in source + key = keySelector(i) + if !haskey(lookup, key) + lookup[key] = Array{TD}(undef, 0) + end + push!(lookup[key], convert(TD, i)) + end + return lookup +end + +function _check_join_key_types(op, ::Type{TKeyOuter}, ::Type{TKeyInner}) where {TKeyOuter,TKeyInner} + if TKeyOuter != TKeyInner + error("The keys in the $op clause have different types, $TKeyOuter and $TKeyInner.") + end +end + +struct EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS<:Function,IKS<:Function,RS<:Function} <: Enumerable + outer::SO + inner::SI + outerKeySelector::OKS + innerKeySelector::IKS + resultSelector::RS + default_inner::TID +end + +Base.eltype(::Type{EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS,IKS,RS}}) where {T,TKey,TID,SO,SI,OKS,IKS,RS} = T + +function left_join(outer::Enumerable, inner::Enumerable, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + TO = eltype(outer) + TI = eltype(inner) + TKeyOuter = Base._return_type(f_outerKeySelector, Tuple{TO,}) + TKeyInner = Base._return_type(f_innerKeySelector, Tuple{TI,}) + + _check_join_key_types("left_join", TKeyOuter, TKeyInner) + + default_inner = _default_value(TI) + TID = typeof(default_inner) + + T = Base._return_type(f_resultSelector, Tuple{TO,TID}) + + SO = typeof(outer) + SI = typeof(inner) + OKS = typeof(f_outerKeySelector) + IKS = typeof(f_innerKeySelector) + RS = typeof(f_resultSelector) + + return EnumerableLeftJoin{T,TKeyOuter,TID,SO,SI,OKS,IKS,RS}(outer, inner, f_outerKeySelector, f_innerKeySelector, f_resultSelector, default_inner) +end + +function Base.iterate(iter::EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS,IKS,RS}) where {T,TKey,TID,SO,SI,OKS,IKS,RS} + results = Array{T}(undef, 0) + + inner_dict = _outerjoin_lookup(iter.inner, iter.innerKeySelector, TKey, TID) + + for i in iter.outer + outerKey = iter.outerKeySelector(i) + if haskey(inner_dict, outerKey) + for j in inner_dict[outerKey] + push!(results, iter.resultSelector(i, j)) + end + else + push!(results, iter.resultSelector(i, iter.default_inner)) + end + end + + if length(results)==0 + return nothing + end + + return results[1], (results, 2) +end + +function Base.iterate(iter::EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS,IKS,RS}, state) where {T,TKey,TID,SO,SI,OKS,IKS,RS} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_rightjoin.jl b/src/enumerable/enumerable_rightjoin.jl new file mode 100644 index 0000000..f5ffd75 --- /dev/null +++ b/src/enumerable/enumerable_rightjoin.jl @@ -0,0 +1,63 @@ +struct EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS<:Function,IKS<:Function,RS<:Function} <: Enumerable + outer::SO + inner::SI + outerKeySelector::OKS + innerKeySelector::IKS + resultSelector::RS + default_outer::TOD +end + +Base.eltype(::Type{EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS,IKS,RS}}) where {T,TKey,TOD,SO,SI,OKS,IKS,RS} = T + +function right_join(outer::Enumerable, inner::Enumerable, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + TO = eltype(outer) + TI = eltype(inner) + TKeyOuter = Base._return_type(f_outerKeySelector, Tuple{TO,}) + TKeyInner = Base._return_type(f_innerKeySelector, Tuple{TI,}) + + _check_join_key_types("right_join", TKeyOuter, TKeyInner) + + default_outer = _default_value(TO) + TOD = typeof(default_outer) + + T = Base._return_type(f_resultSelector, Tuple{TOD,TI}) + + SO = typeof(outer) + SI = typeof(inner) + OKS = typeof(f_outerKeySelector) + IKS = typeof(f_innerKeySelector) + RS = typeof(f_resultSelector) + + return EnumerableRightJoin{T,TKeyInner,TOD,SO,SI,OKS,IKS,RS}(outer, inner, f_outerKeySelector, f_innerKeySelector, f_resultSelector, default_outer) +end + +function Base.iterate(iter::EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS,IKS,RS}) where {T,TKey,TOD,SO,SI,OKS,IKS,RS} + results = Array{T}(undef, 0) + + outer_dict = _outerjoin_lookup(iter.outer, iter.outerKeySelector, TKey, TOD) + + for j in iter.inner + innerKey = iter.innerKeySelector(j) + if haskey(outer_dict, innerKey) + for i in outer_dict[innerKey] + push!(results, iter.resultSelector(i, j)) + end + else + push!(results, iter.resultSelector(iter.default_outer, j)) + end + end + + if length(results)==0 + return nothing + end + + return results[1], (results, 2) +end + +function Base.iterate(iter::EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS,IKS,RS}, state) where {T,TKey,TOD,SO,SI,OKS,IKS,RS} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/operators.jl b/src/operators.jl index 563d3bb..53d7240 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -129,3 +129,12 @@ function pivot_longer end function pivot_wider end function summarize end + +# Outer joins, mirroring Enumerable.LeftJoin/RightJoin/FullJoin (.NET 11). The +# unmatched side is supplied as an all-null element built by `_default_value`. + +function left_join end + +function right_join end + +function full_join end diff --git a/test/runtests.jl b/test/runtests.jl index d5bb042..c99dcb0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,7 @@ using TestItemRunner include("test_core.jl") +include("test_enumerable_outerjoins.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_outerjoins.jl b/test/test_enumerable_outerjoins.jl new file mode 100644 index 0000000..08f3a54 --- /dev/null +++ b/test/test_enumerable_outerjoins.jl @@ -0,0 +1,191 @@ +@testitem "left_join" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=3, x="c")]) + inner = QueryOperators.query([(a=1, y=10), (a=1, y=11), (a=3, y=30)]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 4 + @test res[1] == (x="a", y=DataValue(10)) + @test res[2] == (x="a", y=DataValue(11)) + @test res[3].x == "b" + @test isna(res[3].y) + @test res[4] == (x="c", y=DataValue(30)) + + # Absent values are DataValue, never missing. + @test eltype(res) == NamedTuple{(:x, :y),Tuple{String,DataValue{Int}}} + @test !any(r -> ismissing(r.y), res) +end + +@testitem "left_join keeps every outer row when the inner side is empty" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=2, x="b")]) + inner = QueryOperators.query(NamedTuple{(:a, :y),Tuple{Int,Int}}[]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 2 + @test all(r -> isna(r.y), res) +end + +@testitem "left_join on an empty outer side yields nothing" begin + outer = QueryOperators.query(NamedTuple{(:a, :x),Tuple{Int,String}}[]) + inner = QueryOperators.query([(a=1, y=10)]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 0 +end + +@testitem "left_join rejects mismatched key types" begin + outer = QueryOperators.query([(a=1, x="a")]) + inner = QueryOperators.query([(a="1", y=10)]) + + @test_throws ErrorException QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y))) +end + +@testitem "left_join over scalar elements" begin + using DataValues + + outer = QueryOperators.query([1, 2, 3]) + inner = QueryOperators.query([1, 3]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i, :(i -> i), + i -> i, :(i -> i), + (i, j) -> (o=i, i=j), :((i, j) -> (o=i, i=j)))) + + @test length(res) == 3 + @test res[1] == (o=1, i=DataValue(1)) + @test isna(res[2].i) + @test res[3] == (o=3, i=DataValue(3)) +end + +@testitem "right_join" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=3, x="c")]) + inner = QueryOperators.query([(a=1, y=10), (a=2, y=20), (a=3, y=30)]) + + res = collect(QueryOperators.right_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + # One row per inner element, in inner order. + @test length(res) == 3 + @test res[1] == (x=DataValue("a"), y=10) + @test isna(res[2].x) + @test res[2].y == 20 + @test res[3] == (x=DataValue("c"), y=30) + + @test eltype(res) == NamedTuple{(:x, :y),Tuple{DataValue{String},Int}} + @test !any(r -> ismissing(r.x), res) +end + +@testitem "right_join keeps every inner row when the outer side is empty" begin + using DataValues + + outer = QueryOperators.query(NamedTuple{(:a, :x),Tuple{Int,String}}[]) + inner = QueryOperators.query([(a=1, y=10), (a=2, y=20)]) + + res = collect(QueryOperators.right_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 2 + @test all(r -> isna(r.x), res) +end + +@testitem "full_join" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=2, x="b")]) + inner = QueryOperators.query([(a=1, y=10), (a=3, y=30)]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + # Outer rows first in outer order, then unmatched inner rows. + @test length(res) == 3 + @test res[1] == (x=DataValue("a"), y=DataValue(10)) + @test res[2].x == DataValue("b") + @test isna(res[2].y) + @test isna(res[3].x) + @test res[3].y == DataValue(30) + + @test eltype(res) == NamedTuple{(:x, :y),Tuple{DataValue{String},DataValue{Int}}} + @test !any(r -> ismissing(r.x) || ismissing(r.y), res) +end + +@testitem "full_join with no overlap keeps both sides" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a")]) + inner = QueryOperators.query([(a=2, y=20)]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 2 + @test isna(res[1].y) + @test isna(res[2].x) +end + +@testitem "full_join with duplicate keys on both sides" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=1, x="b")]) + inner = QueryOperators.query([(a=1, y=10), (a=1, y=11)]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + # Cartesian product within the matching key, as for an inner join. + @test length(res) == 4 + @test [(get(r.x), get(r.y)) for r in res] == [("a", 10), ("a", 11), ("b", 10), ("b", 11)] +end + +@testitem "full_join on two empty sides yields nothing" begin + outer = QueryOperators.query(NamedTuple{(:a, :x),Tuple{Int,String}}[]) + inner = QueryOperators.query(NamedTuple{(:a, :y),Tuple{Int,Int}}[]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 0 +end From 841d962332548b6fd3cf4692e0a9ce73188381be Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:07:53 -0700 Subject: [PATCH 2/9] Add the concat, union, except and intersect set operators Ports Enumerable.Concat/Union/UnionBy/Except/ExceptBy/Intersect/IntersectBy. union, except and intersect shadow their Base counterparts, as map, filter, count, take, unique and join already do here. concat is fully lazy and reports a length when both sources have one. union is lazy over both sources behind a growing set of seen keys; except and intersect materialise the second source's keys on first iterate and then walk the first source lazily. All three de-duplicate their output, matching .NET. One deliberate deviation: Enumerable.ExceptBy and IntersectBy take a bare sequence of keys as their second argument, while UnionBy takes a sequence of elements. Here the key selector is applied to both sequences in all three cases. That keeps the operators consistent with each other, matches the shape of the equivalent SQL, and is what a table-shaped second argument makes natural. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 4 + src/enumerable/enumerable.jl | 23 +++ src/enumerable/enumerable_concat.jl | 44 ++++++ src/enumerable/enumerable_except.jl | 62 ++++++++ src/enumerable/enumerable_intersect.jl | 59 ++++++++ src/enumerable/enumerable_union.jl | 58 ++++++++ src/operators.jl | 18 +++ test/runtests.jl | 1 + test/test_enumerable_setops.jl | 188 +++++++++++++++++++++++++ 9 files changed, 457 insertions(+) create mode 100644 src/enumerable/enumerable_concat.jl create mode 100644 src/enumerable/enumerable_except.jl create mode 100644 src/enumerable/enumerable_intersect.jl create mode 100644 src/enumerable/enumerable_union.jl create mode 100644 test/test_enumerable_setops.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index d0fd967..8e1e474 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -26,6 +26,10 @@ include("enumerable/enumerable_count.jl") include("enumerable/enumerable_take.jl") include("enumerable/enumerable_drop.jl") include("enumerable/enumerable_unique.jl") +include("enumerable/enumerable_concat.jl") +include("enumerable/enumerable_union.jl") +include("enumerable/enumerable_except.jl") +include("enumerable/enumerable_intersect.jl") include("enumerable/enumerable_pivot.jl") include("enumerable/enumerable_summarize.jl") include("enumerable/show.jl") diff --git a/src/enumerable/enumerable.jl b/src/enumerable/enumerable.jl index c6469a4..28914a1 100644 --- a/src/enumerable/enumerable.jl +++ b/src/enumerable/enumerable.jl @@ -4,3 +4,26 @@ Base.IteratorSize(::Type{T}) where {T <: Enumerable} = Base.SizeUnknown() IteratorInterfaceExtensions.isiterable(x::Enumerable) = true haslength(S) = Base.IteratorSize(S) isa Union{Base.HasLength, Base.HasShape} ? Base.HasLength() : Base.IteratorSize(S) + +# Operators that walk more than one source keep "which source, how far into it" +# in their iteration state. `_NotStarted` marks a source that has not been +# iterated yet, so that `_iterate_from` can pick the right `iterate` method +# without conflating it with a source whose own state happens to be `nothing`. +struct _NotStarted end + +_iterate_from(source, ::_NotStarted) = iterate(source) +_iterate_from(source, state) = iterate(source, state) + +# Element types of two sources that will be emitted into a single stream have +# to agree, the same requirement `default_if_empty` places on its default value. +function _check_same_eltype(op, ::Type{T1}, ::Type{T2}) where {T1,T2} + if T1 != T2 + error("The two sequences passed to $op have different element types, $T1 and $T2.") + end +end + +function _check_same_keytype(op, ::Type{TKey1}, ::Type{TKey2}) where {TKey1,TKey2} + if TKey1 != TKey2 + error("The keys of the two sequences passed to $op have different types, $TKey1 and $TKey2.") + end +end diff --git a/src/enumerable/enumerable_concat.jl b/src/enumerable/enumerable_concat.jl new file mode 100644 index 0000000..a0b5823 --- /dev/null +++ b/src/enumerable/enumerable_concat.jl @@ -0,0 +1,44 @@ +struct EnumerableConcat{T,S1,S2} <: Enumerable + first::S1 + second::S2 +end + +function concat(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("concat", T1, T2) + + return EnumerableConcat{T1,typeof(first),typeof(second)}(first, second) +end + +Base.eltype(::Type{EnumerableConcat{T,S1,S2}}) where {T,S1,S2} = T + +function Base.IteratorSize(::Type{EnumerableConcat{T,S1,S2}}) where {T,S1,S2} + return haslength(S1) isa Base.HasLength && haslength(S2) isa Base.HasLength ? + Base.HasLength() : Base.SizeUnknown() +end + +Base.length(iter::EnumerableConcat) = length(iter.first) + length(iter.second) + +Base.iterate(iter::EnumerableConcat) = _concat_next(iter, 1, _NotStarted()) + +function Base.iterate(iter::EnumerableConcat, state) + return _concat_next(iter, state.side, state.state) +end + +function _concat_next(iter::EnumerableConcat, side, source_state) + if side == 1 + ret = _iterate_from(iter.first, source_state) + if ret !== nothing + return ret[1], (side=1, state=ret[2]) + end + # First source exhausted — fall through to the second. + side = 2 + source_state = _NotStarted() + end + + ret = _iterate_from(iter.second, source_state) + ret === nothing && return nothing + return ret[1], (side=2, state=ret[2]) +end diff --git a/src/enumerable/enumerable_except.jl b/src/enumerable/enumerable_except.jl new file mode 100644 index 0000000..10f3728 --- /dev/null +++ b/src/enumerable/enumerable_except.jl @@ -0,0 +1,62 @@ +struct EnumerableExcept{T,TKEY,S1,S2,Q<:Function} <: Enumerable + first::S1 + second::S2 + f::Q +end + +Base.eltype(::Type{EnumerableExcept{T,TKEY,S1,S2,Q}}) where {T,TKEY,S1,S2,Q} = T + +function except(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("except", T1, T2) + + return EnumerableExcept{T1,T1,typeof(first),typeof(second),typeof(identity)}(first, second, identity) +end + +# Unlike Enumerable.ExceptBy, whose second argument is a sequence of keys, the +# key selector here is applied to both sequences. That keeps `except_by` +# consistent with `union_by`, matches the shape of the equivalent SQL +# (`WHERE k NOT IN (SELECT k FROM b)`), and is what a table-shaped second +# argument makes natural. +function except_by(first::Enumerable, second::Enumerable, f::Function, f_expr::Expr) + T1 = eltype(first) + T2 = eltype(second) + + TKEY1 = Base._return_type(f, Tuple{T1,}) + TKEY2 = Base._return_type(f, Tuple{T2,}) + + _check_same_keytype("except_by", TKEY1, TKEY2) + + return EnumerableExcept{T1,TKEY1,typeof(first),typeof(second),typeof(f)}(first, second, f) +end + +function Base.iterate(iter::EnumerableExcept{T,TKEY,S1,S2,Q}) where {T,TKEY,S1,S2,Q} + excluded = Set{TKEY}() + for i in iter.second + push!(excluded, iter.f(i)) + end + + return _except_next(iter, excluded, Set{TKEY}(), _NotStarted()) +end + +function Base.iterate(iter::EnumerableExcept, state) + return _except_next(iter, state.excluded, state.observed, state.state) +end + +# Yields the distinct elements of the first source whose key does not occur in +# the second, matching Enumerable.Except's de-duplicating behaviour. +function _except_next(iter::EnumerableExcept, excluded, observed, source_state) + while true + ret = _iterate_from(iter.first, source_state) + ret === nothing && return nothing + + element, source_state = ret + k = iter.f(element) + if !(k in excluded) && !(k in observed) + push!(observed, k) + return element, (excluded=excluded, observed=observed, state=source_state) + end + end +end diff --git a/src/enumerable/enumerable_intersect.jl b/src/enumerable/enumerable_intersect.jl new file mode 100644 index 0000000..1368de1 --- /dev/null +++ b/src/enumerable/enumerable_intersect.jl @@ -0,0 +1,59 @@ +struct EnumerableIntersect{T,TKEY,S1,S2,Q<:Function} <: Enumerable + first::S1 + second::S2 + f::Q +end + +Base.eltype(::Type{EnumerableIntersect{T,TKEY,S1,S2,Q}}) where {T,TKEY,S1,S2,Q} = T + +function intersect(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("intersect", T1, T2) + + return EnumerableIntersect{T1,T1,typeof(first),typeof(second),typeof(identity)}(first, second, identity) +end + +# As with `except_by`, the key selector is applied to both sequences rather than +# the second being a bare sequence of keys as in Enumerable.IntersectBy. +function intersect_by(first::Enumerable, second::Enumerable, f::Function, f_expr::Expr) + T1 = eltype(first) + T2 = eltype(second) + + TKEY1 = Base._return_type(f, Tuple{T1,}) + TKEY2 = Base._return_type(f, Tuple{T2,}) + + _check_same_keytype("intersect_by", TKEY1, TKEY2) + + return EnumerableIntersect{T1,TKEY1,typeof(first),typeof(second),typeof(f)}(first, second, f) +end + +function Base.iterate(iter::EnumerableIntersect{T,TKEY,S1,S2,Q}) where {T,TKEY,S1,S2,Q} + required = Set{TKEY}() + for i in iter.second + push!(required, iter.f(i)) + end + + return _intersect_next(iter, required, Set{TKEY}(), _NotStarted()) +end + +function Base.iterate(iter::EnumerableIntersect, state) + return _intersect_next(iter, state.required, state.observed, state.state) +end + +# Yields the distinct elements of the first source whose key also occurs in the +# second, matching Enumerable.Intersect's de-duplicating behaviour. +function _intersect_next(iter::EnumerableIntersect, required, observed, source_state) + while true + ret = _iterate_from(iter.first, source_state) + ret === nothing && return nothing + + element, source_state = ret + k = iter.f(element) + if k in required && !(k in observed) + push!(observed, k) + return element, (required=required, observed=observed, state=source_state) + end + end +end diff --git a/src/enumerable/enumerable_union.jl b/src/enumerable/enumerable_union.jl new file mode 100644 index 0000000..b884325 --- /dev/null +++ b/src/enumerable/enumerable_union.jl @@ -0,0 +1,58 @@ +struct EnumerableUnion{T,TKEY,S1,S2,Q<:Function} <: Enumerable + first::S1 + second::S2 + f::Q +end + +Base.eltype(::Type{EnumerableUnion{T,TKEY,S1,S2,Q}}) where {T,TKEY,S1,S2,Q} = T + +function union(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("union", T1, T2) + + return EnumerableUnion{T1,T1,typeof(first),typeof(second),typeof(identity)}(first, second, identity) +end + +function union_by(first::Enumerable, second::Enumerable, f::Function, f_expr::Expr) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("union_by", T1, T2) + + TKEY = Base._return_type(f, Tuple{T1,}) + + return EnumerableUnion{T1,TKEY,typeof(first),typeof(second),typeof(f)}(first, second, f) +end + +function Base.iterate(iter::EnumerableUnion{T,TKEY,S1,S2,Q}) where {T,TKEY,S1,S2,Q} + return _union_next(iter, Set{TKEY}(), 1, _NotStarted()) +end + +function Base.iterate(iter::EnumerableUnion, state) + return _union_next(iter, state.observed, state.side, state.state) +end + +# Walks the first source and then the second, yielding each element whose key +# has not been seen before, so that the result is distinct across both sources. +function _union_next(iter::EnumerableUnion{T,TKEY,S1,S2,Q}, observed, side, source_state) where {T,TKEY,S1,S2,Q} + while true + source = side == 1 ? iter.first : iter.second + ret = _iterate_from(source, source_state) + + if ret === nothing + side == 2 && return nothing + side = 2 + source_state = _NotStarted() + continue + end + + element, source_state = ret + k = iter.f(element) + if !(k in observed) + push!(observed, k) + return element, (observed=observed, side=side, state=source_state) + end + end +end diff --git a/src/operators.jl b/src/operators.jl index 53d7240..8b2cc4b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -138,3 +138,21 @@ function left_join end function right_join end function full_join end + +# Set operations. `union`, `except` and `intersect` shadow their Base +# counterparts, as `map`, `filter`, `count`, `take`, `unique` and `join` +# already do in this module. + +function concat end + +function union end + +function union_by end + +function except end + +function except_by end + +function intersect end + +function intersect_by end diff --git a/test/runtests.jl b/test/runtests.jl index c99dcb0..9ab4b33 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,6 +2,7 @@ using TestItemRunner include("test_core.jl") include("test_enumerable_outerjoins.jl") +include("test_enumerable_setops.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_setops.jl b/test/test_enumerable_setops.jl new file mode 100644 index 0000000..d6616a8 --- /dev/null +++ b/test/test_enumerable_setops.jl @@ -0,0 +1,188 @@ +@testitem "concat" begin + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query([3, 4]) + + res = QueryOperators.concat(a, b) + + # Concat keeps duplicates and source order. + @test collect(res) == [1, 2, 3, 3, 4] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 5 +end + +@testitem "concat with an empty side" begin + a = QueryOperators.query([1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.concat(a, empty)) == [1, 2] + @test collect(QueryOperators.concat(empty, a)) == [1, 2] + @test collect(QueryOperators.concat(empty, empty)) == Int[] +end + +@testitem "concat rejects mismatched element types" begin + a = QueryOperators.query([1, 2]) + b = QueryOperators.query(["a"]) + + @test_throws ErrorException QueryOperators.concat(a, b) +end + +@testitem "concat does not touch the second source until the first is exhausted" begin + import IteratorInterfaceExtensions + + # A source that records how many elements have been pulled from it. + mutable struct Counted + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{Counted}) = Int + Base.IteratorSize(::Type{Counted}) = Base.HasLength() + Base.length(c::Counted) = length(c.data) + function Base.iterate(c::Counted, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::Counted) = true + IteratorInterfaceExtensions.getiterator(c::Counted) = c + + second = Counted([10, 20], 0) + res = QueryOperators.concat(QueryOperators.query([1, 2, 3]), QueryOperators.query(second)) + + # Pull two of the three elements of the first source. + it = iterate(res) + it = iterate(res, it[2]) + + @test it[1] == 2 + @test second.pulled == 0 + + # Draining the rest reaches the second source. + @test collect(res) == [1, 2, 3, 10, 20] + @test second.pulled > 0 +end + +@testitem "union" begin + a = QueryOperators.query([1, 2, 2, 3]) + b = QueryOperators.query([3, 4, 4]) + + res = QueryOperators.union(a, b) + + # Distinct across both sources, in first-seen order. + @test collect(res) == [1, 2, 3, 4] + @test eltype(res) == Int +end + +@testitem "union with an empty side" begin + a = QueryOperators.query([1, 1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.union(a, empty)) == [1, 2] + @test collect(QueryOperators.union(empty, a)) == [1, 2] + @test collect(QueryOperators.union(empty, empty)) == Int[] +end + +@testitem "union_by" begin + a = QueryOperators.query([(a=1, x="a"), (a=2, x="b")]) + b = QueryOperators.query([(a=2, x="B"), (a=3, x="c")]) + + res = QueryOperators.union_by(a, b, i -> i.a, :(i -> i.a)) + + # The first element seen for a key wins, so a=2 keeps x="b". + @test collect(res) == [(a=1, x="a"), (a=2, x="b"), (a=3, x="c")] +end + +@testitem "except" begin + a = QueryOperators.query([1, 2, 2, 3, 4]) + b = QueryOperators.query([2, 4]) + + res = QueryOperators.except(a, b) + + # Distinct elements of the first source not present in the second. + @test collect(res) == [1, 3] + @test eltype(res) == Int +end + +@testitem "except de-duplicates the first source" begin + a = QueryOperators.query([1, 1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.except(a, empty)) == [1, 2] +end + +@testitem "except with everything excluded" begin + a = QueryOperators.query([1, 2]) + b = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.except(a, b)) == Int[] +end + +@testitem "except_by" begin + a = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=3, x="c")]) + b = QueryOperators.query([(a=2, x="ignored")]) + + res = QueryOperators.except_by(a, b, i -> i.a, :(i -> i.a)) + + @test collect(res) == [(a=1, x="a"), (a=3, x="c")] +end + +@testitem "intersect" begin + a = QueryOperators.query([1, 2, 2, 3, 4]) + b = QueryOperators.query([2, 4, 5]) + + res = QueryOperators.intersect(a, b) + + # Distinct elements of the first source that also occur in the second, + # in first-source order. + @test collect(res) == [2, 4] + @test eltype(res) == Int +end + +@testitem "intersect with no overlap" begin + a = QueryOperators.query([1, 2]) + b = QueryOperators.query([3, 4]) + + @test collect(QueryOperators.intersect(a, b)) == Int[] +end + +@testitem "intersect with an empty side" begin + a = QueryOperators.query([1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.intersect(a, empty)) == Int[] + @test collect(QueryOperators.intersect(empty, a)) == Int[] +end + +@testitem "intersect_by" begin + a = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=2, x="bb")]) + b = QueryOperators.query([(a=2, x="ignored"), (a=9, x="ignored")]) + + res = QueryOperators.intersect_by(a, b, i -> i.a, :(i -> i.a)) + + # Distinct by key, so only the first a=2 element is kept. + @test collect(res) == [(a=2, x="b")] +end + +@testitem "set operators over DataValue elements treat nulls as equal" begin + using DataValues + + a = QueryOperators.query([DataValue{Int}(), DataValue(1), DataValue{Int}()]) + b = QueryOperators.query([DataValue(1)]) + + # Two nulls are a single distinct element, and 1 is excluded by the second + # sequence, so a single null survives. + res = collect(QueryOperators.except(a, b)) + @test length(res) == 1 + @test isna(res[1]) +end + +@testitem "set operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=1, v=2), (k=2, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + res = QueryOperators.union_by(grouped, grouped, g -> QueryOperators.key(g), :(g -> key(g))) + + # Unioning a grouping with itself by key is the identity on the groups. + collected = collect(res) + @test length(collected) == 2 + @test [QueryOperators.key(g) for g in collected] == [1, 2] +end From 2a6da18d01e654520d38352f1be715cd9844977d Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:09:59 -0700 Subject: [PATCH 3/9] Add the order, reverse, shuffle and index operators Ports Enumerable.Order and OrderDescending (.NET 7), Enumerable.Shuffle (.NET 10) and Enumerable.Index (.NET 9), plus Enumerable.Reverse. order and order_descending are expressed in terms of orderby with an identity key selector, so they produce an EnumerableOrderby and thenby can still follow them. index yields (index, item) NamedTuples with 1-based indices, matching Julia rather than .NET's 0-based Index(). It is lazy; reverse and shuffle have to collect their source first. shuffle takes an optional RNG, which makes it reproducible; without one it uses the default RNG and, as in .NET, is not cryptographically secure. This adds Random to the dependencies. Co-Authored-By: Claude Opus 5 --- Project.toml | 2 + src/QueryOperators.jl | 4 + src/enumerable/enumerable_index.jl | 33 +++++++ src/enumerable/enumerable_orderby.jl | 7 ++ src/enumerable/enumerable_reverse.jl | 34 +++++++ src/enumerable/enumerable_shuffle.jl | 41 ++++++++ src/operators.jl | 14 +++ test/runtests.jl | 1 + test/test_enumerable_ordering.jl | 141 +++++++++++++++++++++++++++ 9 files changed, 277 insertions(+) create mode 100644 src/enumerable/enumerable_index.jl create mode 100644 src/enumerable/enumerable_reverse.jl create mode 100644 src/enumerable/enumerable_shuffle.jl create mode 100644 test/test_enumerable_ordering.jl diff --git a/Project.toml b/Project.toml index e1382fc..7d1bf77 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" TableShowUtils = "5e66a065-1f0a-5976-b372-e0b8c017ca10" DataValues = "e7dc6d0d-1eca-5fa6-8ad6-5aecde8b7ea5" IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [extras] TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" @@ -18,6 +19,7 @@ DataStructures = "0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19" TableShowUtils = "0.1.1, 0.2, 0.3, 1" DataValues = "0.4.4, 0.5, 1" IteratorInterfaceExtensions = "0.1.1, 1" +Random = "1" [targets] test = ["Test", "TestItemRunner"] diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index 8e1e474..d2c46db 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -4,6 +4,7 @@ using DataStructures using IteratorInterfaceExtensions using TableShowUtils import DataValues +import Random export Grouping, key @@ -30,6 +31,9 @@ include("enumerable/enumerable_concat.jl") include("enumerable/enumerable_union.jl") include("enumerable/enumerable_except.jl") include("enumerable/enumerable_intersect.jl") +include("enumerable/enumerable_reverse.jl") +include("enumerable/enumerable_shuffle.jl") +include("enumerable/enumerable_index.jl") include("enumerable/enumerable_pivot.jl") include("enumerable/enumerable_summarize.jl") include("enumerable/show.jl") diff --git a/src/enumerable/enumerable_index.jl b/src/enumerable/enumerable_index.jl new file mode 100644 index 0000000..997dcf5 --- /dev/null +++ b/src/enumerable/enumerable_index.jl @@ -0,0 +1,33 @@ +struct EnumerableIndex{T,TI,S} <: Enumerable + source::S +end + +# Enumerable.Index (.NET 9) pairs each element with its position. .NET yields +# (Index, Item) tuples; the Julia equivalent is an (index, item) NamedTuple. +# Indices are 1-based, matching the rest of Julia rather than .NET. +function index(source::Enumerable) + TI = eltype(source) + T = NamedTuple{(:index, :item),Tuple{Int,TI}} + return EnumerableIndex{T,TI,typeof(source)}(source) +end + +Base.IteratorSize(::Type{EnumerableIndex{T,TI,S}}) where {T,TI,S} = haslength(S) + +Base.eltype(::Type{EnumerableIndex{T,TI,S}}) where {T,TI,S} = T + +Base.length(iter::EnumerableIndex) = length(iter.source) + +function Base.iterate(iter::EnumerableIndex{T,TI,S}) where {T,TI,S} + ret = iterate(iter.source) + ret === nothing && return nothing + + return T((1, ret[1])), (i=1, state=ret[2]) +end + +function Base.iterate(iter::EnumerableIndex{T,TI,S}, state) where {T,TI,S} + ret = iterate(iter.source, state.state) + ret === nothing && return nothing + + i = state.i + 1 + return T((i, ret[1])), (i=i, state=ret[2]) +end diff --git a/src/enumerable/enumerable_orderby.jl b/src/enumerable/enumerable_orderby.jl index b815d7e..496400b 100644 --- a/src/enumerable/enumerable_orderby.jl +++ b/src/enumerable/enumerable_orderby.jl @@ -28,6 +28,13 @@ function orderby_descending(source::Enumerable, f::Function, f_expr::Expr) return EnumerableOrderby{T,typeof(source),KS,TKS}(source, f, true) end +# Enumerable.Order/OrderDescending (.NET 7): sort by the elements themselves. +# Expressed in terms of orderby so that `thenby` can still follow them — it +# walks back up the source chain looking for an EnumerableOrderby. +order(source::Enumerable) = orderby(source, identity, :(i -> i)) + +order_descending(source::Enumerable) = orderby_descending(source, identity, :(i -> i)) + function Base.iterate(iter::EnumerableOrderby{T,S,KS,TKS}) where {T,S,KS,TKS} rows = (Base.IteratorSize(typeof(iter)) isa Base.HasLength || Base.IteratorSize(typeof(iter)) isa Base.HasShape) ? length(iter) : 0 diff --git a/src/enumerable/enumerable_reverse.jl b/src/enumerable/enumerable_reverse.jl new file mode 100644 index 0000000..bf4e681 --- /dev/null +++ b/src/enumerable/enumerable_reverse.jl @@ -0,0 +1,34 @@ +struct EnumerableReverse{T,S} <: Enumerable + source::S +end + +function reverse(source::Enumerable) + T = eltype(source) + return EnumerableReverse{T,typeof(source)}(source) +end + +Base.IteratorSize(::Type{EnumerableReverse{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableReverse{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableReverse) = length(iter.source) + +# Reversing needs the whole source, so the elements are collected on the first +# call and then handed out back to front. +function Base.iterate(iter::EnumerableReverse{T,S}) where {T,S} + elements = Base.collect(T, iter.source) + + if length(elements)==0 + return nothing + end + + return elements[end], (elements, length(elements)-1) +end + +function Base.iterate(iter::EnumerableReverse{T,S}, state) where {T,S} + if state[2]<1 + return nothing + else + return state[1][state[2]], (state[1], state[2]-1) + end +end diff --git a/src/enumerable/enumerable_shuffle.jl b/src/enumerable/enumerable_shuffle.jl new file mode 100644 index 0000000..2b32a31 --- /dev/null +++ b/src/enumerable/enumerable_shuffle.jl @@ -0,0 +1,41 @@ +struct EnumerableShuffle{T,S,R} <: Enumerable + source::S + rng::R +end + +function shuffle(source::Enumerable) + return shuffle(source, Random.default_rng()) +end + +function shuffle(source::Enumerable, rng::Random.AbstractRNG) + T = eltype(source) + return EnumerableShuffle{T,typeof(source),typeof(rng)}(source, rng) +end + +Base.IteratorSize(::Type{EnumerableShuffle{T,S,R}}) where {T,S,R} = haslength(S) + +Base.eltype(::Type{EnumerableShuffle{T,S,R}}) where {T,S,R} = T + +Base.length(iter::EnumerableShuffle) = length(iter.source) + +# Like Enumerable.Shuffle, the randomisation is not cryptographically secure. +# Pass an explicit RNG to make a shuffle reproducible. +function Base.iterate(iter::EnumerableShuffle{T,S,R}) where {T,S,R} + elements = Base.collect(T, iter.source) + + if length(elements)==0 + return nothing + end + + Random.shuffle!(iter.rng, elements) + + return elements[1], (elements, 2) +end + +function Base.iterate(iter::EnumerableShuffle{T,S,R}, state) where {T,S,R} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/operators.jl b/src/operators.jl index 8b2cc4b..108731b 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -156,3 +156,17 @@ function except_by end function intersect end function intersect_by end + +# Ordering and row position. `order`/`order_descending` are Enumerable.Order and +# OrderDescending (.NET 7); `shuffle` is Enumerable.Shuffle (.NET 10); `index` +# is Enumerable.Index (.NET 9). + +function order end + +function order_descending end + +function reverse end + +function shuffle end + +function index end diff --git a/test/runtests.jl b/test/runtests.jl index 9ab4b33..b5c33b7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,6 +3,7 @@ using TestItemRunner include("test_core.jl") include("test_enumerable_outerjoins.jl") include("test_enumerable_setops.jl") +include("test_enumerable_ordering.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_ordering.jl b/test/test_enumerable_ordering.jl new file mode 100644 index 0000000..269fe7c --- /dev/null +++ b/test/test_enumerable_ordering.jl @@ -0,0 +1,141 @@ +@testitem "order" begin + source = QueryOperators.query([3, 1, 2]) + + res = QueryOperators.order(source) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "order_descending" begin + source = QueryOperators.query([3, 1, 2]) + + @test collect(QueryOperators.order_descending(source)) == [3, 2, 1] +end + +@testitem "order on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.order(source)) == Int[] + @test collect(QueryOperators.order_descending(source)) == Int[] +end + +@testitem "thenby can follow order" begin + source = QueryOperators.query([(a=1, b=2), (a=1, b=1), (a=0, b=9)]) + + # order sorts by the whole element; thenby then refines it, which only + # works because order reuses EnumerableOrderby. + res = QueryOperators.@thenby(QueryOperators.order(source), i -> i.b) + + @test collect(res) == [(a=0, b=9), (a=1, b=1), (a=1, b=2)] +end + +@testitem "reverse" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.reverse(source) + + @test collect(res) == [3, 2, 1] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "reverse on empty and single-element sources" begin + @test collect(QueryOperators.reverse(QueryOperators.query(Int[]))) == Int[] + @test collect(QueryOperators.reverse(QueryOperators.query([7]))) == [7] +end + +@testitem "reverse of reverse is the identity" begin + source = QueryOperators.query([1, 2, 3, 4]) + + @test collect(QueryOperators.reverse(QueryOperators.reverse(source))) == [1, 2, 3, 4] +end + +@testitem "shuffle is a permutation of its source" begin + using Random + + source = QueryOperators.query(collect(1:50)) + + res = QueryOperators.shuffle(source) + + @test sort(collect(res)) == collect(1:50) + @test eltype(res) == Int + @test length(res) == 50 +end + +@testitem "shuffle with an explicit rng is reproducible" begin + using Random + + source = QueryOperators.query(collect(1:50)) + + a = collect(QueryOperators.shuffle(source, MersenneTwister(42))) + b = collect(QueryOperators.shuffle(source, MersenneTwister(42))) + + @test a == b + @test sort(a) == collect(1:50) +end + +@testitem "shuffle on an empty source" begin + @test collect(QueryOperators.shuffle(QueryOperators.query(Int[]))) == Int[] +end + +@testitem "index" begin + source = QueryOperators.query(["a", "b", "c"]) + + res = QueryOperators.index(source) + + # 1-based, matching Julia rather than .NET's 0-based Index(). + @test collect(res) == [(index=1, item="a"), (index=2, item="b"), (index=3, item="c")] + @test eltype(res) == NamedTuple{(:index, :item),Tuple{Int,String}} + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "index on an empty source" begin + source = QueryOperators.query(String[]) + + @test collect(QueryOperators.index(source)) == NamedTuple{(:index, :item),Tuple{Int,String}}[] +end + +@testitem "index is lazy" begin + import IteratorInterfaceExtensions + + mutable struct CountedIndexSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedIndexSource}) = Int + Base.IteratorSize(::Type{CountedIndexSource}) = Base.HasLength() + Base.length(c::CountedIndexSource) = length(c.data) + function Base.iterate(c::CountedIndexSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedIndexSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedIndexSource) = c + + src = CountedIndexSource([1, 2, 3, 4, 5], 0) + res = QueryOperators.index(QueryOperators.query(src)) + + it = iterate(res) + it = iterate(res, it[2]) + + @test it[1] == (index=2, item=2) + @test src.pulled == 2 +end + +@testitem "ordering operators work downstream of groupby" begin + source = QueryOperators.query([(k=2, v=1), (k=1, v=2), (k=3, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + reversed = collect(QueryOperators.reverse(grouped)) + @test [QueryOperators.key(g) for g in reversed] == [3, 1, 2] + + indexed = collect(QueryOperators.index(grouped)) + @test [i.index for i in indexed] == [1, 2, 3] + @test [QueryOperators.key(i.item) for i in indexed] == [2, 1, 3] +end From fefe33389038b5ebb2205a25a1b2d8f6de79050f Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:11:46 -0700 Subject: [PATCH 4/9] Add the count_by, aggregate_by and chunk operators Ports Enumerable.CountBy and AggregateBy (.NET 9) and Enumerable.Chunk (.NET 6). count_by and aggregate_by name their key columns through the existing _key_namedtuple, so they agree with summarize: a scalar key becomes a column called `key`, a NamedTuple key contributes one column per field. A test asserts count_by and groupby-plus-summarize produce identical output. summarize remains the more general and more idiomatic way to aggregate; aggregate_by is here for LINQ parity, and its accumulator takes (accumulated, element) to match .NET's argument order. chunk pulls at most n elements per call, so a chunked source is only walked as far as the batches actually consumed. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 3 + src/enumerable/enumerable_aggregateby.jl | 49 +++++++ src/enumerable/enumerable_chunk.jl | 42 ++++++ src/enumerable/enumerable_countby.jl | 44 ++++++ src/operators.jl | 10 ++ test/runtests.jl | 1 + test/test_enumerable_keyed_aggregation.jl | 165 ++++++++++++++++++++++ 7 files changed, 314 insertions(+) create mode 100644 src/enumerable/enumerable_aggregateby.jl create mode 100644 src/enumerable/enumerable_chunk.jl create mode 100644 src/enumerable/enumerable_countby.jl create mode 100644 test/test_enumerable_keyed_aggregation.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index d2c46db..dc1c575 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -36,6 +36,9 @@ include("enumerable/enumerable_shuffle.jl") include("enumerable/enumerable_index.jl") include("enumerable/enumerable_pivot.jl") include("enumerable/enumerable_summarize.jl") +include("enumerable/enumerable_countby.jl") +include("enumerable/enumerable_aggregateby.jl") +include("enumerable/enumerable_chunk.jl") include("enumerable/show.jl") include("source_iterable.jl") diff --git a/src/enumerable/enumerable_aggregateby.jl b/src/enumerable/enumerable_aggregateby.jl new file mode 100644 index 0000000..b185365 --- /dev/null +++ b/src/enumerable/enumerable_aggregateby.jl @@ -0,0 +1,49 @@ +_aggregate_by_row(k, v) = Base.merge(_key_namedtuple(k), (value = v,)) + +struct EnumerableAggregateBy{T,TKey,TACC,S,Q<:Function,A<:Function} <: Enumerable + source::S + f::Q + seed::TACC + accumulator::A +end + +Base.eltype(::Type{EnumerableAggregateBy{T,TKey,TACC,S,Q,A}}) where {T,TKey,TACC,S,Q,A} = T + +# Enumerable.AggregateBy (.NET 9): fold the elements of each key into a single +# value, without materialising the intermediate groupings. `accumulator` is +# called as `accumulator(accumulated, element)`, matching .NET's argument order. +# +# `summarize` is the more general and more idiomatic way to aggregate here; +# `aggregate_by` exists for LINQ parity. +function aggregate_by(source::Enumerable, f::Function, f_expr::Expr, seed, accumulator::Function) + TS = eltype(source) + TKey = Base._return_type(f, Tuple{TS,}) + TACC = typeof(seed) + T = Base._return_type(_aggregate_by_row, Tuple{TKey,TACC}) + + return EnumerableAggregateBy{T,TKey,TACC,typeof(source),typeof(f),typeof(accumulator)}(source, f, seed, accumulator) +end + +function Base.iterate(iter::EnumerableAggregateBy{T,TKey,TACC,S,Q,A}) where {T,TKey,TACC,S,Q,A} + accumulated = OrderedDict{TKey,TACC}() + for i in iter.source + k = iter.f(i) + accumulated[k] = iter.accumulator(get(accumulated, k, iter.seed), i) + end + + rows = T[_aggregate_by_row(k, v) for (k, v) in accumulated] + + if length(rows)==0 + return nothing + end + + return rows[1], (rows, 2) +end + +function Base.iterate(iter::EnumerableAggregateBy{T,TKey,TACC,S,Q,A}, state) where {T,TKey,TACC,S,Q,A} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_chunk.jl b/src/enumerable/enumerable_chunk.jl new file mode 100644 index 0000000..caf7075 --- /dev/null +++ b/src/enumerable/enumerable_chunk.jl @@ -0,0 +1,42 @@ +struct EnumerableChunk{T,TI,S} <: Enumerable + source::S + n::Int +end + +# Enumerable.Chunk (.NET 6): split the source into batches of at most `n` +# elements. The final batch is shorter when the source does not divide evenly. +function chunk(source::Enumerable, n::Integer) + n < 1 && error("The chunk size must be at least 1, got $n.") + + TI = eltype(source) + T = Vector{TI} + + return EnumerableChunk{T,TI,typeof(source)}(source, Int(n)) +end + +Base.IteratorSize(::Type{EnumerableChunk{T,TI,S}}) where {T,TI,S} = haslength(S) + +Base.eltype(::Type{EnumerableChunk{T,TI,S}}) where {T,TI,S} = T + +Base.length(iter::EnumerableChunk) = cld(length(iter.source), iter.n) + +Base.iterate(iter::EnumerableChunk) = _chunk_next(iter, _NotStarted()) + +Base.iterate(iter::EnumerableChunk, state) = _chunk_next(iter, state) + +# Pulls at most `n` elements per call, so a chunked source is only walked as +# far as the batches actually consumed. +function _chunk_next(iter::EnumerableChunk{T,TI,S}, source_state) where {T,TI,S} + buffer = TI[] + + while length(buffer) < iter.n + ret = _iterate_from(iter.source, source_state) + ret === nothing && break + push!(buffer, ret[1]) + source_state = ret[2] + end + + length(buffer)==0 && return nothing + + return buffer, source_state +end diff --git a/src/enumerable/enumerable_countby.jl b/src/enumerable/enumerable_countby.jl new file mode 100644 index 0000000..3217aad --- /dev/null +++ b/src/enumerable/enumerable_countby.jl @@ -0,0 +1,44 @@ +# Key columns are named exactly as `summarize` names them: a scalar key becomes +# a column called `key`, a NamedTuple key contributes one column per field. +_count_by_row(k, n::Int) = Base.merge(_key_namedtuple(k), (count = n,)) + +struct EnumerableCountBy{T,TKey,S,Q<:Function} <: Enumerable + source::S + f::Q +end + +Base.eltype(::Type{EnumerableCountBy{T,TKey,S,Q}}) where {T,TKey,S,Q} = T + +# Enumerable.CountBy (.NET 9): the frequency of each key, without materialising +# the intermediate groupings that `groupby` would build. +function count_by(source::Enumerable, f::Function, f_expr::Expr) + TS = eltype(source) + TKey = Base._return_type(f, Tuple{TS,}) + T = Base._return_type(_count_by_row, Tuple{TKey,Int}) + + return EnumerableCountBy{T,TKey,typeof(source),typeof(f)}(source, f) +end + +function Base.iterate(iter::EnumerableCountBy{T,TKey,S,Q}) where {T,TKey,S,Q} + counts = OrderedDict{TKey,Int}() + for i in iter.source + k = iter.f(i) + counts[k] = get(counts, k, 0) + 1 + end + + rows = T[_count_by_row(k, n) for (k, n) in counts] + + if length(rows)==0 + return nothing + end + + return rows[1], (rows, 2) +end + +function Base.iterate(iter::EnumerableCountBy{T,TKey,S,Q}, state) where {T,TKey,S,Q} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/operators.jl b/src/operators.jl index 108731b..c004a5e 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -170,3 +170,13 @@ function reverse end function shuffle end function index end + +# Keyed aggregation and batching. `count_by` and `aggregate_by` are +# Enumerable.CountBy and AggregateBy (.NET 9); `chunk` is Enumerable.Chunk +# (.NET 6). + +function count_by end + +function aggregate_by end + +function chunk end diff --git a/test/runtests.jl b/test/runtests.jl index b5c33b7..9d6837a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ include("test_core.jl") include("test_enumerable_outerjoins.jl") include("test_enumerable_setops.jl") include("test_enumerable_ordering.jl") +include("test_enumerable_keyed_aggregation.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_keyed_aggregation.jl b/test/test_enumerable_keyed_aggregation.jl new file mode 100644 index 0000000..15d78ff --- /dev/null +++ b/test/test_enumerable_keyed_aggregation.jl @@ -0,0 +1,165 @@ +@testitem "count_by with a scalar key" begin + source = QueryOperators.query([(k="a", v=1), (k="b", v=2), (k="a", v=3)]) + + res = QueryOperators.count_by(source, i -> i.k, :(i -> i.k)) + + # A scalar key becomes a column called `key`, as summarize names it. + @test collect(res) == [(key="a", count=2), (key="b", count=1)] + @test eltype(res) == NamedTuple{(:key, :count),Tuple{String,Int}} +end + +@testitem "count_by with a NamedTuple key splats the key columns" begin + source = QueryOperators.query([(a=1, b=1), (a=1, b=1), (a=1, b=2)]) + + res = QueryOperators.count_by(source, i -> (a=i.a, b=i.b), :(i -> (a=i.a, b=i.b))) + + @test collect(res) == [(a=1, b=1, count=2), (a=1, b=2, count=1)] +end + +@testitem "count_by preserves first-seen key order" begin + source = QueryOperators.query([3, 1, 3, 2, 1]) + + res = QueryOperators.count_by(source, i -> i, :(i -> i)) + + @test collect(res) == [(key=3, count=2), (key=1, count=2), (key=2, count=1)] +end + +@testitem "count_by on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.count_by(source, i -> i, :(i -> i))) == NamedTuple{(:key, :count),Tuple{Int,Int}}[] +end + +@testitem "count_by agrees with groupby plus summarize" begin + using Statistics + + data = [(k=1, v=10), (k=2, v=20), (k=1, v=30), (k=3, v=40), (k=1, v=50)] + + by_count = collect(QueryOperators.count_by(QueryOperators.query(data), i -> i.k, :(i -> i.k))) + + grouped = QueryOperators.@groupby_simple(QueryOperators.query(data), i -> i.k) + by_summarize = collect(QueryOperators.summarize( + grouped, + g -> Base.merge(QueryOperators._key_namedtuple(QueryOperators.key(g)), (count = length(g),)), + :(g -> (count = length(g),)))) + + @test by_count == by_summarize +end + +@testitem "aggregate_by" begin + source = QueryOperators.query([(id="0", score=42), (id="1", score=5), (id="2", score=4), (id="1", score=10), (id="0", score=25)]) + + res = QueryOperators.aggregate_by( + source, + i -> i.id, :(i -> i.id), + 0, + (total, cur) -> total + cur.score) + + # The example from the .NET 9 release notes. + @test collect(res) == [(key="0", value=67), (key="1", value=15), (key="2", value=4)] + @test eltype(res) == NamedTuple{(:key, :value),Tuple{String,Int}} +end + +@testitem "aggregate_by with a NamedTuple key" begin + source = QueryOperators.query([(a=1, b=1, v=2), (a=1, b=1, v=3), (a=2, b=1, v=5)]) + + res = QueryOperators.aggregate_by( + source, + i -> (a=i.a, b=i.b), :(i -> (a=i.a, b=i.b)), + 1, + (acc, cur) -> acc * cur.v) + + @test collect(res) == [(a=1, b=1, value=6), (a=2, b=1, value=5)] +end + +@testitem "aggregate_by on an empty source" begin + source = QueryOperators.query(Int[]) + + res = QueryOperators.aggregate_by(source, i -> i, :(i -> i), 0, (acc, cur) -> acc + cur) + + @test collect(res) == NamedTuple{(:key, :value),Tuple{Int,Int}}[] +end + +@testitem "aggregate_by never shares the seed between keys" begin + source = QueryOperators.query([1, 1, 2, 2, 2]) + + res = QueryOperators.aggregate_by(source, i -> i, :(i -> i), Int[], (acc, cur) -> vcat(acc, cur)) + + @test collect(res) == [(key=1, value=[1, 1]), (key=2, value=[2, 2, 2])] +end + +@testitem "chunk" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + res = QueryOperators.chunk(source, 2) + + # The final chunk is short when the source does not divide evenly. + @test collect(res) == [[1, 2], [3, 4], [5]] + @test eltype(res) == Vector{Int} + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "chunk when the size divides evenly" begin + source = QueryOperators.query([1, 2, 3, 4]) + + @test collect(QueryOperators.chunk(source, 2)) == [[1, 2], [3, 4]] + @test length(QueryOperators.chunk(source, 2)) == 2 +end + +@testitem "chunk with a size larger than the source" begin + source = QueryOperators.query([1, 2]) + + @test collect(QueryOperators.chunk(source, 10)) == [[1, 2]] +end + +@testitem "chunk on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.chunk(source, 3)) == Vector{Int}[] +end + +@testitem "chunk rejects a size below 1" begin + source = QueryOperators.query([1, 2, 3]) + + @test_throws ErrorException QueryOperators.chunk(source, 0) + @test_throws ErrorException QueryOperators.chunk(source, -1) +end + +@testitem "chunk only walks the source as far as the batches consumed" begin + import IteratorInterfaceExtensions + + mutable struct CountedChunkSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedChunkSource}) = Int + Base.IteratorSize(::Type{CountedChunkSource}) = Base.HasLength() + Base.length(c::CountedChunkSource) = length(c.data) + function Base.iterate(c::CountedChunkSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedChunkSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedChunkSource) = c + + src = CountedChunkSource(collect(1:100), 0) + res = QueryOperators.chunk(QueryOperators.query(src), 3) + + it = iterate(res) + + @test it[1] == [1, 2, 3] + @test src.pulled == 3 +end + +@testitem "chunk works downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2), (k=3, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + chunks = collect(QueryOperators.chunk(grouped, 2)) + + @test length(chunks) == 2 + @test [QueryOperators.key(g) for g in chunks[1]] == [1, 2] + @test [QueryOperators.key(g) for g in chunks[2]] == [3] +end From 99ed76e6cac7a859ce97d07ba186aa734f22c281 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:13:32 -0700 Subject: [PATCH 5/9] Add the take_while, drop_while, take_last and drop_last operators Ports Enumerable.TakeWhile, SkipWhile, TakeLast and SkipLast. The Skip operators are named drop_* to match the existing `drop` rather than LINQ's `Skip`. take_while and drop_while are lazy: take_while stops at the first element that fails the predicate and never examines the rest, and drop_while passes everything through untested once the leading run is gone. take_last holds at most n elements in a CircularBuffer while walking the source. drop_last stays n elements behind the source, emitting an element only once n further ones have been read. A count of zero or less yields nothing for take_last and leaves the source unchanged for drop_last, matching .NET. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 4 + src/enumerable/enumerable_droplast.jl | 48 +++++++ src/enumerable/enumerable_dropwhile.jl | 27 ++++ src/enumerable/enumerable_takelast.jl | 42 +++++++ src/enumerable/enumerable_takewhile.jl | 23 ++++ src/operators.jl | 11 ++ test/runtests.jl | 1 + test/test_enumerable_partitioning.jl | 165 +++++++++++++++++++++++++ 8 files changed, 321 insertions(+) create mode 100644 src/enumerable/enumerable_droplast.jl create mode 100644 src/enumerable/enumerable_dropwhile.jl create mode 100644 src/enumerable/enumerable_takelast.jl create mode 100644 src/enumerable/enumerable_takewhile.jl create mode 100644 test/test_enumerable_partitioning.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index dc1c575..5f77374 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -26,6 +26,10 @@ include("enumerable/enumerable_defaultifempty.jl") include("enumerable/enumerable_count.jl") include("enumerable/enumerable_take.jl") include("enumerable/enumerable_drop.jl") +include("enumerable/enumerable_takewhile.jl") +include("enumerable/enumerable_dropwhile.jl") +include("enumerable/enumerable_takelast.jl") +include("enumerable/enumerable_droplast.jl") include("enumerable/enumerable_unique.jl") include("enumerable/enumerable_concat.jl") include("enumerable/enumerable_union.jl") diff --git a/src/enumerable/enumerable_droplast.jl b/src/enumerable/enumerable_droplast.jl new file mode 100644 index 0000000..90885ab --- /dev/null +++ b/src/enumerable/enumerable_droplast.jl @@ -0,0 +1,48 @@ +struct EnumerableDropLast{T,S} <: Enumerable + source::S + n::Int +end + +# Enumerable.SkipLast: everything but the trailing `n` elements. A count of +# zero or less leaves the source unchanged, as in .NET. +function drop_last(source::Enumerable, n::Integer) + T = eltype(source) + return EnumerableDropLast{T,typeof(source)}(source, max(Int(n), 0)) +end + +Base.IteratorSize(::Type{EnumerableDropLast{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableDropLast{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableDropLast) = max(length(iter.source) - iter.n, 0) + +# Stays `n` elements behind the source: an element is only emitted once `n` +# further elements have been read, which proves it is not one of the last `n`. +function Base.iterate(iter::EnumerableDropLast{T,S}) where {T,S} + buffer = T[] + source_state = _NotStarted() + + while length(buffer) < iter.n + ret = _iterate_from(iter.source, source_state) + # Fewer than n elements in total, so every one of them is dropped. + ret === nothing && return nothing + push!(buffer, ret[1]) + source_state = ret[2] + end + + return _drop_last_next(iter, buffer, source_state) +end + +function Base.iterate(iter::EnumerableDropLast, state) + return _drop_last_next(iter, state.buffer, state.state) +end + +function _drop_last_next(iter::EnumerableDropLast, buffer, source_state) + ret = _iterate_from(iter.source, source_state) + ret === nothing && return nothing + + push!(buffer, ret[1]) + element = popfirst!(buffer) + + return element, (buffer=buffer, state=ret[2]) +end diff --git a/src/enumerable/enumerable_dropwhile.jl b/src/enumerable/enumerable_dropwhile.jl new file mode 100644 index 0000000..d7a8897 --- /dev/null +++ b/src/enumerable/enumerable_dropwhile.jl @@ -0,0 +1,27 @@ +struct EnumerableDropWhile{T,S,Q<:Function} <: Enumerable + source::S + f::Q +end + +# Enumerable.SkipWhile: discards the leading run of elements satisfying the +# predicate, then yields everything that follows without testing it again. +function drop_while(source::Enumerable, f::Function, f_expr::Expr) + T = eltype(source) + return EnumerableDropWhile{T,typeof(source),typeof(f)}(source, f) +end + +Base.eltype(::Type{EnumerableDropWhile{T,S,Q}}) where {T,S,Q} = T + +function Base.iterate(iter::EnumerableDropWhile) + ret = iterate(iter.source) + + while ret !== nothing && iter.f(ret[1]) + ret = iterate(iter.source, ret[2]) + end + + ret === nothing && return nothing + + return ret[1], ret[2] +end + +Base.iterate(iter::EnumerableDropWhile, state) = iterate(iter.source, state) diff --git a/src/enumerable/enumerable_takelast.jl b/src/enumerable/enumerable_takelast.jl new file mode 100644 index 0000000..6ff0c98 --- /dev/null +++ b/src/enumerable/enumerable_takelast.jl @@ -0,0 +1,42 @@ +struct EnumerableTakeLast{T,S} <: Enumerable + source::S + n::Int +end + +# Enumerable.TakeLast: the trailing `n` elements. A count of zero or less +# yields nothing, as in .NET. +function take_last(source::Enumerable, n::Integer) + T = eltype(source) + return EnumerableTakeLast{T,typeof(source)}(source, max(Int(n), 0)) +end + +Base.IteratorSize(::Type{EnumerableTakeLast{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableTakeLast{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableTakeLast) = min(length(iter.source), iter.n) + +# Which elements are last is only known once the source is exhausted, so the +# whole source is walked, holding at most `n` elements in a ring buffer. +function Base.iterate(iter::EnumerableTakeLast{T,S}) where {T,S} + iter.n == 0 && return nothing + + buffer = CircularBuffer{T}(iter.n) + for i in iter.source + push!(buffer, i) + end + + length(buffer)==0 && return nothing + + elements = Base.collect(buffer) + + return elements[1], (elements, 2) +end + +function Base.iterate(iter::EnumerableTakeLast{T,S}, state) where {T,S} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_takewhile.jl b/src/enumerable/enumerable_takewhile.jl new file mode 100644 index 0000000..59cfe43 --- /dev/null +++ b/src/enumerable/enumerable_takewhile.jl @@ -0,0 +1,23 @@ +struct EnumerableTakeWhile{T,S,Q<:Function} <: Enumerable + source::S + f::Q +end + +# Enumerable.TakeWhile: yields elements until the predicate first fails, then +# stops — later elements are not examined even if they would satisfy it. +function take_while(source::Enumerable, f::Function, f_expr::Expr) + T = eltype(source) + return EnumerableTakeWhile{T,typeof(source),typeof(f)}(source, f) +end + +Base.eltype(::Type{EnumerableTakeWhile{T,S,Q}}) where {T,S,Q} = T + +Base.iterate(iter::EnumerableTakeWhile) = _take_while_step(iter, iterate(iter.source)) + +Base.iterate(iter::EnumerableTakeWhile, state) = _take_while_step(iter, iterate(iter.source, state)) + +function _take_while_step(iter::EnumerableTakeWhile, ret) + ret === nothing && return nothing + iter.f(ret[1]) || return nothing + return ret[1], ret[2] +end diff --git a/src/operators.jl b/src/operators.jl index c004a5e..c8bc382 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -180,3 +180,14 @@ function count_by end function aggregate_by end function chunk end + +# Partitioning. `drop_while` and `drop_last` are Enumerable.SkipWhile and +# SkipLast, named to match the existing `drop` rather than LINQ's `Skip`. + +function take_while end + +function drop_while end + +function take_last end + +function drop_last end diff --git a/test/runtests.jl b/test/runtests.jl index 9d6837a..03469de 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,6 +5,7 @@ include("test_enumerable_outerjoins.jl") include("test_enumerable_setops.jl") include("test_enumerable_ordering.jl") include("test_enumerable_keyed_aggregation.jl") +include("test_enumerable_partitioning.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_partitioning.jl b/test/test_enumerable_partitioning.jl new file mode 100644 index 0000000..668b979 --- /dev/null +++ b/test/test_enumerable_partitioning.jl @@ -0,0 +1,165 @@ +@testitem "take_while" begin + source = QueryOperators.query([1, 2, 3, 4, 1, 2]) + + res = QueryOperators.take_while(source, i -> i < 3, :(i -> i < 3)) + + # Stops at the first failure; the trailing 1 and 2 are not taken. + @test collect(res) == [1, 2] + @test eltype(res) == Int +end + +@testitem "take_while taking everything or nothing" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.take_while(source, i -> true, :(i -> true))) == [1, 2, 3] + @test collect(QueryOperators.take_while(source, i -> false, :(i -> false))) == Int[] +end + +@testitem "take_while on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.take_while(source, i -> true, :(i -> true))) == Int[] +end + +@testitem "take_while stops walking the source at the first failure" begin + import IteratorInterfaceExtensions + + mutable struct CountedTakeWhileSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedTakeWhileSource}) = Int + Base.IteratorSize(::Type{CountedTakeWhileSource}) = Base.HasLength() + Base.length(c::CountedTakeWhileSource) = length(c.data) + function Base.iterate(c::CountedTakeWhileSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedTakeWhileSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedTakeWhileSource) = c + + src = CountedTakeWhileSource(collect(1:100), 0) + res = QueryOperators.take_while(QueryOperators.query(src), i -> i < 4, :(i -> i < 4)) + + @test collect(res) == [1, 2, 3] + # Three taken plus the one that failed the predicate. + @test src.pulled == 4 +end + +@testitem "drop_while" begin + source = QueryOperators.query([1, 2, 3, 4, 1, 2]) + + res = QueryOperators.drop_while(source, i -> i < 3, :(i -> i < 3)) + + # Only the leading run is dropped; the trailing 1 and 2 survive. + @test collect(res) == [3, 4, 1, 2] + @test eltype(res) == Int +end + +@testitem "drop_while dropping everything or nothing" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.drop_while(source, i -> true, :(i -> true))) == Int[] + @test collect(QueryOperators.drop_while(source, i -> false, :(i -> false))) == [1, 2, 3] +end + +@testitem "drop_while on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.drop_while(source, i -> true, :(i -> true))) == Int[] +end + +@testitem "take_while and drop_while partition the source" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + p = i -> i < 3 + + taken = collect(QueryOperators.take_while(source, p, :(i -> i < 3))) + dropped = collect(QueryOperators.drop_while(source, p, :(i -> i < 3))) + + @test vcat(taken, dropped) == [1, 2, 3, 4, 5] +end + +@testitem "take_last" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + res = QueryOperators.take_last(source, 2) + + @test collect(res) == [4, 5] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 2 +end + +@testitem "take_last with a count at or beyond the source length" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.take_last(source, 3)) == [1, 2, 3] + @test collect(QueryOperators.take_last(source, 10)) == [1, 2, 3] + @test length(QueryOperators.take_last(source, 10)) == 3 +end + +@testitem "take_last with a count of zero or less" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.take_last(source, 0)) == Int[] + @test collect(QueryOperators.take_last(source, -1)) == Int[] +end + +@testitem "take_last on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.take_last(source, 2)) == Int[] +end + +@testitem "drop_last" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + res = QueryOperators.drop_last(source, 2) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "drop_last with a count at or beyond the source length" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.drop_last(source, 3)) == Int[] + @test collect(QueryOperators.drop_last(source, 10)) == Int[] + @test length(QueryOperators.drop_last(source, 10)) == 0 +end + +@testitem "drop_last with a count of zero or less is the identity" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.drop_last(source, 0)) == [1, 2, 3] + @test collect(QueryOperators.drop_last(source, -1)) == [1, 2, 3] +end + +@testitem "drop_last on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.drop_last(source, 2)) == Int[] +end + +@testitem "take_last and drop_last partition the source" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + dropped = collect(QueryOperators.drop_last(source, 2)) + taken = collect(QueryOperators.take_last(source, 2)) + + @test vcat(dropped, taken) == [1, 2, 3, 4, 5] +end + +@testitem "partitioning operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2), (k=3, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + last_two = collect(QueryOperators.take_last(grouped, 2)) + @test [QueryOperators.key(g) for g in last_two] == [2, 3] + + all_but_last = collect(QueryOperators.drop_last(grouped, 1)) + @test [QueryOperators.key(g) for g in all_but_last] == [1, 2] +end From 4389dfac415ed0c90fc25789ce27baf7d072e3b0 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:15:09 -0700 Subject: [PATCH 6/9] Add the append, prepend and zip operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports Enumerable.Append, Prepend and Zip. append and zip shadow their Base counterparts. append and prepend convert the extra element to the source's element type, so appending an Int to a sequence of Float64 works rather than erroring. zip stops at the shorter of its two sources, matching both Base.zip and .NET. Nothing is padded, so zip manufactures no null values — a test asserts the result contains neither missing nor DataValue. Without a result selector it yields Tuples, as Base.zip and Enumerable.Zip both do. All three are lazy, and zip pulls at most one element past the end of the shorter source. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 3 + src/enumerable/enumerable_append.jl | 32 ++++++ src/enumerable/enumerable_prepend.jl | 25 +++++ src/enumerable/enumerable_zip.jl | 46 ++++++++ src/operators.jl | 8 ++ test/runtests.jl | 1 + test/test_enumerable_combining.jl | 154 +++++++++++++++++++++++++++ 7 files changed, 269 insertions(+) create mode 100644 src/enumerable/enumerable_append.jl create mode 100644 src/enumerable/enumerable_prepend.jl create mode 100644 src/enumerable/enumerable_zip.jl create mode 100644 test/test_enumerable_combining.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index 5f77374..0e951fd 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -38,6 +38,9 @@ include("enumerable/enumerable_intersect.jl") include("enumerable/enumerable_reverse.jl") include("enumerable/enumerable_shuffle.jl") include("enumerable/enumerable_index.jl") +include("enumerable/enumerable_append.jl") +include("enumerable/enumerable_prepend.jl") +include("enumerable/enumerable_zip.jl") include("enumerable/enumerable_pivot.jl") include("enumerable/enumerable_summarize.jl") include("enumerable/enumerable_countby.jl") diff --git a/src/enumerable/enumerable_append.jl b/src/enumerable/enumerable_append.jl new file mode 100644 index 0000000..de92291 --- /dev/null +++ b/src/enumerable/enumerable_append.jl @@ -0,0 +1,32 @@ +struct EnumerableAppend{T,S} <: Enumerable + source::S + element::T +end + +# Enumerable.Append: the source followed by one more element. The element is +# converted to the source's element type, so appending an Int to a sequence of +# Float64 works. +function append(source::Enumerable, element) + T = eltype(source) + return EnumerableAppend{T,typeof(source)}(source, convert(T, element)) +end + +Base.IteratorSize(::Type{EnumerableAppend{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableAppend{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableAppend) = length(iter.source) + 1 + +Base.iterate(iter::EnumerableAppend) = _append_next(iter, _NotStarted()) + +function Base.iterate(iter::EnumerableAppend, state) + # `state.done` marks the appended element as already handed out. + state.done && return nothing + return _append_next(iter, state.state) +end + +function _append_next(iter::EnumerableAppend, source_state) + ret = _iterate_from(iter.source, source_state) + ret === nothing && return iter.element, (done=true, state=source_state) + return ret[1], (done=false, state=ret[2]) +end diff --git a/src/enumerable/enumerable_prepend.jl b/src/enumerable/enumerable_prepend.jl new file mode 100644 index 0000000..7a643e4 --- /dev/null +++ b/src/enumerable/enumerable_prepend.jl @@ -0,0 +1,25 @@ +struct EnumerablePrepend{T,S} <: Enumerable + source::S + element::T +end + +# Enumerable.Prepend: one element followed by the source. As with `append`, the +# element is converted to the source's element type. +function prepend(source::Enumerable, element) + T = eltype(source) + return EnumerablePrepend{T,typeof(source)}(source, convert(T, element)) +end + +Base.IteratorSize(::Type{EnumerablePrepend{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerablePrepend{T,S}}) where {T,S} = T + +Base.length(iter::EnumerablePrepend) = length(iter.source) + 1 + +Base.iterate(iter::EnumerablePrepend) = (iter.element, (state=_NotStarted(),)) + +function Base.iterate(iter::EnumerablePrepend, state) + ret = _iterate_from(iter.source, state.state) + ret === nothing && return nothing + return ret[1], (state=ret[2],) +end diff --git a/src/enumerable/enumerable_zip.jl b/src/enumerable/enumerable_zip.jl new file mode 100644 index 0000000..e80dd58 --- /dev/null +++ b/src/enumerable/enumerable_zip.jl @@ -0,0 +1,46 @@ +struct EnumerableZip{T,S1,S2,RS<:Function} <: Enumerable + first::S1 + second::S2 + resultSelector::RS +end + +Base.eltype(::Type{EnumerableZip{T,S1,S2,RS}}) where {T,S1,S2,RS} = T + +# Enumerable.Zip. Like Base.zip and .NET, the result stops at the shorter of +# the two sources — nothing is padded, so no null values are manufactured. +function zip(first::Enumerable, second::Enumerable) + return _zip(first, second, tuple) +end + +function zip(first::Enumerable, second::Enumerable, f_resultSelector::Function, resultSelector::Expr) + return _zip(first, second, f_resultSelector) +end + +function _zip(first::Enumerable, second::Enumerable, f_resultSelector::Function) + T1 = eltype(first) + T2 = eltype(second) + T = Base._return_type(f_resultSelector, Tuple{T1,T2}) + + return EnumerableZip{T,typeof(first),typeof(second),typeof(f_resultSelector)}(first, second, f_resultSelector) +end + +function Base.IteratorSize(::Type{EnumerableZip{T,S1,S2,RS}}) where {T,S1,S2,RS} + return haslength(S1) isa Base.HasLength && haslength(S2) isa Base.HasLength ? + Base.HasLength() : Base.SizeUnknown() +end + +Base.length(iter::EnumerableZip) = min(length(iter.first), length(iter.second)) + +Base.iterate(iter::EnumerableZip) = _zip_next(iter, _NotStarted(), _NotStarted()) + +Base.iterate(iter::EnumerableZip, state) = _zip_next(iter, state.s1, state.s2) + +function _zip_next(iter::EnumerableZip, s1, s2) + r1 = _iterate_from(iter.first, s1) + r1 === nothing && return nothing + + r2 = _iterate_from(iter.second, s2) + r2 === nothing && return nothing + + return iter.resultSelector(r1[1], r2[1]), (s1=r1[2], s2=r2[2]) +end diff --git a/src/operators.jl b/src/operators.jl index c8bc382..ff25395 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -191,3 +191,11 @@ function drop_while end function take_last end function drop_last end + +# Combining sequences. `append` and `zip` shadow their Base counterparts. + +function append end + +function prepend end + +function zip end diff --git a/test/runtests.jl b/test/runtests.jl index 03469de..db5e852 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,6 +6,7 @@ include("test_enumerable_setops.jl") include("test_enumerable_ordering.jl") include("test_enumerable_keyed_aggregation.jl") include("test_enumerable_partitioning.jl") +include("test_enumerable_combining.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_combining.jl b/test/test_enumerable_combining.jl new file mode 100644 index 0000000..48e56b4 --- /dev/null +++ b/test/test_enumerable_combining.jl @@ -0,0 +1,154 @@ +@testitem "append" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.append(source, 4) + + @test collect(res) == [1, 2, 3, 4] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 4 +end + +@testitem "append to an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.append(source, 1)) == [1] +end + +@testitem "append converts the element to the source element type" begin + source = QueryOperators.query([1.0, 2.0]) + + res = QueryOperators.append(source, 3) + + @test collect(res) == [1.0, 2.0, 3.0] + @test eltype(res) == Float64 +end + +@testitem "append rejects an element that cannot be converted" begin + source = QueryOperators.query([1, 2]) + + @test_throws MethodError QueryOperators.append(source, "three") +end + +@testitem "append of NamedTuple rows" begin + source = QueryOperators.query([(a=1, b="x")]) + + res = QueryOperators.append(source, (a=2, b="y")) + + @test collect(res) == [(a=1, b="x"), (a=2, b="y")] +end + +@testitem "prepend" begin + source = QueryOperators.query([2, 3]) + + res = QueryOperators.prepend(source, 1) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "prepend to an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.prepend(source, 1)) == [1] +end + +@testitem "append and prepend compose" begin + source = QueryOperators.query([2, 3]) + + res = QueryOperators.append(QueryOperators.prepend(source, 1), 4) + + @test collect(res) == [1, 2, 3, 4] + @test length(res) == 4 +end + +@testitem "zip" begin + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query(["a", "b", "c"]) + + res = QueryOperators.zip(a, b) + + @test collect(res) == [(1, "a"), (2, "b"), (3, "c")] + @test eltype(res) == Tuple{Int,String} + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "zip truncates to the shorter source" begin + a = QueryOperators.query([1, 2, 3, 4]) + b = QueryOperators.query(["a", "b"]) + + @test collect(QueryOperators.zip(a, b)) == [(1, "a"), (2, "b")] + @test length(QueryOperators.zip(a, b)) == 2 + @test collect(QueryOperators.zip(b, a)) == [("a", 1), ("b", 2)] +end + +@testitem "zip with an empty source" begin + a = QueryOperators.query([1, 2]) + empty = QueryOperators.query(String[]) + + @test collect(QueryOperators.zip(a, empty)) == Tuple{Int,String}[] + @test collect(QueryOperators.zip(empty, a)) == Tuple{String,Int}[] +end + +@testitem "zip with a result selector" begin + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query([10, 20, 30]) + + res = QueryOperators.zip(a, b, (x, y) -> (sum=x + y,), :((x, y) -> (sum=x + y,))) + + @test collect(res) == [(sum=11,), (sum=22,), (sum=33,)] + @test eltype(res) == NamedTuple{(:sum,),Tuple{Int}} +end + +@testitem "zip pads nothing, so it manufactures no null values" begin + using DataValues + + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query([10]) + + res = collect(QueryOperators.zip(a, b)) + + @test length(res) == 1 + @test !any(r -> any(ismissing, r), res) + @test !any(r -> any(x -> x isa DataValue, r), res) +end + +@testitem "zip does not over-read the longer source" begin + import IteratorInterfaceExtensions + + mutable struct CountedZipSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedZipSource}) = Int + Base.IteratorSize(::Type{CountedZipSource}) = Base.HasLength() + Base.length(c::CountedZipSource) = length(c.data) + function Base.iterate(c::CountedZipSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedZipSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedZipSource) = c + + long = CountedZipSource(collect(1:100), 0) + res = QueryOperators.zip(QueryOperators.query(long), QueryOperators.query([1, 2])) + + @test length(collect(res)) == 2 + # Three pulls: two matched, plus the one that had no partner. + @test long.pulled == 3 +end + +@testitem "combining operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + zipped = collect(QueryOperators.zip(grouped, QueryOperators.query(["first", "second"]))) + + @test length(zipped) == 2 + @test [QueryOperators.key(g) for (g, _) in zipped] == [1, 2] + @test [s for (_, s) in zipped] == ["first", "second"] +end From fac9fbb1dbd27c09330fdf5ae130025b6f0e1a2a Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:17:38 -0700 Subject: [PATCH 7/9] Add the terminal operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports Enumerable.MinBy and MaxBy (.NET 6), Any, All, Contains, SequenceEqual, Aggregate, First, Last, Single and ElementAt. These return a value rather than another Enumerable, so a query ending in one is executed immediately — `count` was already the sole member of that family here. min_by and max_by return the element, not the key, and keep the first element on ties as .NET does. contains and sequence_equal compare with isequal, so DataValue nulls match each other, consistent with unique and the set operators. element_at is 1-based, matching the new `index` operator and the rest of Julia rather than .NET's 0-based ElementAt. summarize remains the idiomatic way to aggregate a table; aggregate is here for LINQ parity. Defining `any` in this module captured the Base.any call in enumerable_pivot.jl, which is now qualified. A test covers that regression. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 1 + src/enumerable/enumerable_pivot.jl | 3 +- src/enumerable/enumerable_terminal.jl | 193 ++++++++++++++++++++++++++ src/operators.jl | 26 ++++ test/runtests.jl | 1 + test/test_enumerable_terminal.jl | 191 +++++++++++++++++++++++++ 6 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 src/enumerable/enumerable_terminal.jl create mode 100644 test/test_enumerable_terminal.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index 0e951fd..06e658c 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -46,6 +46,7 @@ include("enumerable/enumerable_summarize.jl") include("enumerable/enumerable_countby.jl") include("enumerable/enumerable_aggregateby.jl") include("enumerable/enumerable_chunk.jl") +include("enumerable/enumerable_terminal.jl") include("enumerable/show.jl") include("source_iterable.jl") diff --git a/src/enumerable/enumerable_pivot.jl b/src/enumerable/enumerable_pivot.jl index c38e2eb..1832018 100644 --- a/src/enumerable/enumerable_pivot.jl +++ b/src/enumerable/enumerable_pivot.jl @@ -165,7 +165,8 @@ end include_ops = (:include_name, :include_position, :include_startswith, :include_endswith, :include_occursin, :include_all, :include_range, :include_range_idx) - has_positive = any(inst[1] ∈ include_ops for inst in instructions) + # Base.any, not the `any` query operator this module also defines. + has_positive = Base.any(inst[1] ∈ include_ops for inst in instructions) result = has_positive ? Symbol[] : copy(all_names) diff --git a/src/enumerable/enumerable_terminal.jl b/src/enumerable/enumerable_terminal.jl new file mode 100644 index 0000000..1ea702b --- /dev/null +++ b/src/enumerable/enumerable_terminal.jl @@ -0,0 +1,193 @@ +# Terminal operators. Unlike every other operator here these return a value +# rather than another Enumerable, so a query that ends in one is executed +# immediately. `count` in enumerable_count.jl is the pre-existing member of +# this family. + +_no_elements(op) = error("$op was called on a sequence with no elements.") + +_no_match(op) = error("$op was called on a sequence with no element matching the predicate.") + +# --- Quantifiers --- + +any(source::Enumerable) = iterate(source) !== nothing + +function any(source::Enumerable, f::Function, f_expr::Expr) + for i in source + f(i) && return true + end + return false +end + +function all(source::Enumerable, f::Function, f_expr::Expr) + for i in source + f(i) || return false + end + return true +end + +# Compares with isequal rather than ==, so that DataValue nulls match each +# other and NaN matches NaN, consistent with how `unique` and the set +# operators identify elements. +function contains(source::Enumerable, value) + for i in source + isequal(i, value) && return true + end + return false +end + +function sequence_equal(a::Enumerable, b::Enumerable) + sa = iterate(a) + sb = iterate(b) + + while sa !== nothing && sb !== nothing + isequal(sa[1], sb[1]) || return false + sa = iterate(a, sa[2]) + sb = iterate(b, sb[2]) + end + + # Equal only if both ran out at the same point. + return sa === nothing && sb === nothing +end + +# --- Extremes --- + +# Enumerable.MinBy/MaxBy (.NET 6) return the element itself, not the key. Ties +# keep the first such element, as in .NET, because the comparison is strict. +min_by(source::Enumerable, f::Function, f_expr::Expr) = + _extreme_by(source, f, isless, "min_by") + +max_by(source::Enumerable, f::Function, f_expr::Expr) = + _extreme_by(source, f, (candidate, best) -> isless(best, candidate), "max_by") + +function _extreme_by(source::Enumerable, f::Function, better::Function, op) + ret = iterate(source) + ret === nothing && _no_elements(op) + + best = ret[1] + best_key = f(best) + + ret = iterate(source, ret[2]) + while ret !== nothing + candidate_key = f(ret[1]) + if better(candidate_key, best_key) + best = ret[1] + best_key = candidate_key + end + ret = iterate(source, ret[2]) + end + + return best +end + +# --- Folds --- + +# Enumerable.Aggregate without a seed folds from the first element and requires +# a non-empty sequence. `summarize` is the idiomatic way to aggregate a table. +function aggregate(source::Enumerable, f::Function, f_expr::Expr) + ret = iterate(source) + ret === nothing && _no_elements("aggregate") + + accumulated = ret[1] + ret = iterate(source, ret[2]) + while ret !== nothing + accumulated = f(accumulated, ret[1]) + ret = iterate(source, ret[2]) + end + + return accumulated +end + +function aggregate(source::Enumerable, seed, f::Function, f_expr::Expr) + accumulated = seed + for i in source + accumulated = f(accumulated, i) + end + return accumulated +end + +# --- Element access --- + +function first(source::Enumerable) + ret = iterate(source) + ret === nothing && _no_elements("first") + return ret[1] +end + +function first(source::Enumerable, f::Function, f_expr::Expr) + for i in source + f(i) && return i + end + _no_match("first") +end + +function last(source::Enumerable) + ret = iterate(source) + ret === nothing && _no_elements("last") + + element = ret[1] + ret = iterate(source, ret[2]) + while ret !== nothing + element = ret[1] + ret = iterate(source, ret[2]) + end + + return element +end + +function last(source::Enumerable, f::Function, f_expr::Expr) + element = Base.Ref{eltype(source)}() + found = false + + for i in source + if f(i) + element[] = i + found = true + end + end + + found || _no_match("last") + + return element[] +end + +function single(source::Enumerable) + ret = iterate(source) + ret === nothing && _no_elements("single") + + element = ret[1] + iterate(source, ret[2]) === nothing || + error("single was called on a sequence with more than one element.") + + return element +end + +function single(source::Enumerable, f::Function, f_expr::Expr) + element = Base.Ref{eltype(source)}() + found = false + + for i in source + if f(i) + found && error("single was called on a sequence with more than one element matching the predicate.") + element[] = i + found = true + end + end + + found || _no_match("single") + + return element[] +end + +# 1-based, matching `index` and the rest of Julia rather than .NET's 0-based +# ElementAt. +function element_at(source::Enumerable, n::Integer) + n < 1 && error("element_at was called with index $n; the index must be at least 1.") + + seen = 0 + for i in source + seen += 1 + seen == n && return i + end + + error("element_at was called with index $n on a sequence with only $seen elements.") +end diff --git a/src/operators.jl b/src/operators.jl index ff25395..d2ae886 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -199,3 +199,29 @@ function append end function prepend end function zip end + +# Terminal operators, which return a value rather than another Enumerable. +# `count` above is the pre-existing member of this family. Several of these +# shadow Base functions of the same name. + +function min_by end + +function max_by end + +function any end + +function all end + +function contains end + +function sequence_equal end + +function aggregate end + +function first end + +function last end + +function single end + +function element_at end diff --git a/test/runtests.jl b/test/runtests.jl index db5e852..3a1c997 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,6 +7,7 @@ include("test_enumerable_ordering.jl") include("test_enumerable_keyed_aggregation.jl") include("test_enumerable_partitioning.jl") include("test_enumerable_combining.jl") +include("test_enumerable_terminal.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_terminal.jl b/test/test_enumerable_terminal.jl new file mode 100644 index 0000000..b3bbd96 --- /dev/null +++ b/test/test_enumerable_terminal.jl @@ -0,0 +1,191 @@ +@testitem "any" begin + @test QueryOperators.any(QueryOperators.query([1, 2, 3])) == true + @test QueryOperators.any(QueryOperators.query(Int[])) == false + + source = QueryOperators.query([1, 2, 3]) + @test QueryOperators.any(source, i -> i > 2, :(i -> i > 2)) == true + @test QueryOperators.any(source, i -> i > 9, :(i -> i > 9)) == false + @test QueryOperators.any(QueryOperators.query(Int[]), i -> true, :(i -> true)) == false +end + +@testitem "all" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.all(source, i -> i > 0, :(i -> i > 0)) == true + @test QueryOperators.all(source, i -> i > 1, :(i -> i > 1)) == false + + # Vacuously true on an empty sequence, as in .NET. + @test QueryOperators.all(QueryOperators.query(Int[]), i -> false, :(i -> false)) == true +end + +@testitem "contains" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.contains(source, 2) == true + @test QueryOperators.contains(source, 9) == false + @test QueryOperators.contains(QueryOperators.query(Int[]), 1) == false +end + +@testitem "contains compares with isequal" begin + using DataValues + + @test QueryOperators.contains(QueryOperators.query([NaN, 1.0]), NaN) == true + + nulls = QueryOperators.query([DataValue{Int}(), DataValue(1)]) + @test QueryOperators.contains(nulls, DataValue{Int}()) == true +end + +@testitem "sequence_equal" begin + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2, 3]), QueryOperators.query([1, 2, 3])) == true + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2, 3]), QueryOperators.query([1, 2])) == false + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2]), QueryOperators.query([1, 2, 3])) == false + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2]), QueryOperators.query([2, 1])) == false + @test QueryOperators.sequence_equal(QueryOperators.query(Int[]), QueryOperators.query(Int[])) == true +end + +@testitem "min_by and max_by" begin + source = QueryOperators.query([(a=2, x="b"), (a=1, x="a"), (a=3, x="c")]) + + # The whole element is returned, not the key. + @test QueryOperators.min_by(source, i -> i.a, :(i -> i.a)) == (a=1, x="a") + @test QueryOperators.max_by(source, i -> i.a, :(i -> i.a)) == (a=3, x="c") +end + +@testitem "min_by and max_by keep the first element on ties" begin + source = QueryOperators.query([(a=1, x="first"), (a=1, x="second")]) + + @test QueryOperators.min_by(source, i -> i.a, :(i -> i.a)) == (a=1, x="first") + @test QueryOperators.max_by(source, i -> i.a, :(i -> i.a)) == (a=1, x="first") +end + +@testitem "min_by and max_by reject an empty source" begin + source = QueryOperators.query(Int[]) + + @test_throws ErrorException QueryOperators.min_by(source, i -> i, :(i -> i)) + @test_throws ErrorException QueryOperators.max_by(source, i -> i, :(i -> i)) +end + +@testitem "aggregate without a seed" begin + source = QueryOperators.query([1, 2, 3, 4]) + + @test QueryOperators.aggregate(source, (a, b) -> a + b, :((a, b) -> a + b)) == 10 + @test QueryOperators.aggregate(QueryOperators.query([7]), (a, b) -> a + b, :((a, b) -> a + b)) == 7 +end + +@testitem "aggregate without a seed rejects an empty source" begin + source = QueryOperators.query(Int[]) + + @test_throws ErrorException QueryOperators.aggregate(source, (a, b) -> a + b, :((a, b) -> a + b)) +end + +@testitem "aggregate with a seed" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.aggregate(source, 100, (a, b) -> a + b, :((a, b) -> a + b)) == 106 + + # The seed alone is the result for an empty source. + @test QueryOperators.aggregate(QueryOperators.query(Int[]), 100, (a, b) -> a + b, :((a, b) -> a + b)) == 100 +end + +@testitem "aggregate with a seed of a different type" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.aggregate(source, "", (acc, cur) -> acc * string(cur), :((acc, cur) -> acc * string(cur))) + + @test res == "123" +end + +@testitem "first" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.first(source) == 1 + @test QueryOperators.first(source, i -> i > 1, :(i -> i > 1)) == 2 +end + +@testitem "first errors on an empty source or no match" begin + @test_throws ErrorException QueryOperators.first(QueryOperators.query(Int[])) + @test_throws ErrorException QueryOperators.first(QueryOperators.query([1, 2]), i -> i > 9, :(i -> i > 9)) +end + +@testitem "last" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.last(source) == 3 + @test QueryOperators.last(source, i -> i < 3, :(i -> i < 3)) == 2 +end + +@testitem "last errors on an empty source or no match" begin + @test_throws ErrorException QueryOperators.last(QueryOperators.query(Int[])) + @test_throws ErrorException QueryOperators.last(QueryOperators.query([1, 2]), i -> i > 9, :(i -> i > 9)) +end + +@testitem "last over NamedTuple rows" begin + source = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=3, x="c")]) + + @test QueryOperators.last(source) == (a=3, x="c") + @test QueryOperators.last(source, i -> i.a < 3, :(i -> i.a < 3)) == (a=2, x="b") +end + +@testitem "single" begin + @test QueryOperators.single(QueryOperators.query([7])) == 7 + @test QueryOperators.single(QueryOperators.query([1, 2, 3]), i -> i == 2, :(i -> i == 2)) == 2 +end + +@testitem "single errors unless exactly one element matches" begin + @test_throws ErrorException QueryOperators.single(QueryOperators.query(Int[])) + @test_throws ErrorException QueryOperators.single(QueryOperators.query([1, 2])) + + source = QueryOperators.query([1, 2, 3]) + @test_throws ErrorException QueryOperators.single(source, i -> i > 1, :(i -> i > 1)) + @test_throws ErrorException QueryOperators.single(source, i -> i > 9, :(i -> i > 9)) +end + +@testitem "element_at" begin + source = QueryOperators.query(["a", "b", "c"]) + + # 1-based, matching `index` and the rest of Julia. + @test QueryOperators.element_at(source, 1) == "a" + @test QueryOperators.element_at(source, 3) == "c" +end + +@testitem "element_at rejects out-of-range indices" begin + source = QueryOperators.query([1, 2, 3]) + + @test_throws ErrorException QueryOperators.element_at(source, 0) + @test_throws ErrorException QueryOperators.element_at(source, -1) + @test_throws ErrorException QueryOperators.element_at(source, 4) + @test_throws ErrorException QueryOperators.element_at(QueryOperators.query(Int[]), 1) +end + +@testitem "terminal operators compose with other operators" begin + source = QueryOperators.query([1, 2, 3, 4, 5, 6]) + + evens = QueryOperators.@filter(source, i -> i % 2 == 0) + + @test QueryOperators.first(evens) == 2 + @test QueryOperators.last(evens) == 6 + @test QueryOperators.aggregate(evens, (a, b) -> a + b, :((a, b) -> a + b)) == 12 + @test QueryOperators.any(evens) == true + @test QueryOperators.element_at(evens, 2) == 4 +end + +@testitem "terminal operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2), (k=2, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + biggest = QueryOperators.max_by(grouped, g -> length(g), :(g -> length(g))) + + @test QueryOperators.key(biggest) == 2 + @test length(biggest) == 2 + @test QueryOperators.count(grouped) == 2 +end + +@testitem "pivot_longer still works now that QueryOperators defines any" begin + # enumerable_pivot.jl calls Base.any on a generator; defining a query + # operator called `any` in the same module must not capture that call. + source = QueryOperators.query([(year=2017, US=1, EU=2), (year=2018, US=3, EU=4)]) + + res = QueryOperators.pivot_longer(source, (:US, :EU)) + + @test length(collect(res)) == 4 +end From e4091bd784c82497ac7d149d04380c367ad86c50 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:18:33 -0700 Subject: [PATCH 8/9] Add the of_type and cast operators Ports Enumerable.OfType and Cast. of_type keeps the elements that are instances of T and narrows the element type to T, which is useful for sources whose element type is Any or a Union and a no-op filter on a homogeneously typed table. .NET's Cast is a type assertion; the Julia counterpart is a conversion, so cast runs every element through convert and fails the way convert would on an element that cannot be represented as T. Co-Authored-By: Claude Opus 5 --- src/QueryOperators.jl | 1 + src/enumerable/enumerable_oftype.jl | 55 +++++++++++++++++ src/operators.jl | 6 ++ test/runtests.jl | 1 + test/test_enumerable_typefiltering.jl | 89 +++++++++++++++++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 src/enumerable/enumerable_oftype.jl create mode 100644 test/test_enumerable_typefiltering.jl diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index 06e658c..ca610e7 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -41,6 +41,7 @@ include("enumerable/enumerable_index.jl") include("enumerable/enumerable_append.jl") include("enumerable/enumerable_prepend.jl") include("enumerable/enumerable_zip.jl") +include("enumerable/enumerable_oftype.jl") include("enumerable/enumerable_pivot.jl") include("enumerable/enumerable_summarize.jl") include("enumerable/enumerable_countby.jl") diff --git a/src/enumerable/enumerable_oftype.jl b/src/enumerable/enumerable_oftype.jl new file mode 100644 index 0000000..e675e1f --- /dev/null +++ b/src/enumerable/enumerable_oftype.jl @@ -0,0 +1,55 @@ +struct EnumerableOfType{T,S} <: Enumerable + source::S +end + +# Enumerable.OfType: keeps only the elements that are instances of T, and +# narrows the element type to T. Useful for sources whose element type is Any +# or a Union; a no-op filter on a homogeneously typed table. +function of_type(source::Enumerable, ::Type{T}) where {T} + return EnumerableOfType{T,typeof(source)}(source) +end + +Base.eltype(::Type{EnumerableOfType{T,S}}) where {T,S} = T + +Base.iterate(iter::EnumerableOfType) = _of_type_next(iter, _NotStarted()) + +Base.iterate(iter::EnumerableOfType, state) = _of_type_next(iter, state) + +function _of_type_next(iter::EnumerableOfType{T,S}, source_state) where {T,S} + while true + ret = _iterate_from(iter.source, source_state) + ret === nothing && return nothing + + element, source_state = ret + element isa T && return element, source_state + end +end + +struct EnumerableCast{T,S} <: Enumerable + source::S +end + +# Enumerable.Cast. .NET's Cast is a type assertion; the Julia counterpart is a +# conversion, so `cast` runs every element through `convert` and fails the same +# way `convert` would on an element that cannot be represented as T. +function cast(source::Enumerable, ::Type{T}) where {T} + return EnumerableCast{T,typeof(source)}(source) +end + +Base.IteratorSize(::Type{EnumerableCast{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableCast{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableCast) = length(iter.source) + +function Base.iterate(iter::EnumerableCast{T,S}) where {T,S} + ret = iterate(iter.source) + ret === nothing && return nothing + return convert(T, ret[1]), ret[2] +end + +function Base.iterate(iter::EnumerableCast{T,S}, state) where {T,S} + ret = iterate(iter.source, state) + ret === nothing && return nothing + return convert(T, ret[1]), ret[2] +end diff --git a/src/operators.jl b/src/operators.jl index d2ae886..74a3f45 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -225,3 +225,9 @@ function last end function single end function element_at end + +# Type filtering. + +function of_type end + +function cast end diff --git a/test/runtests.jl b/test/runtests.jl index 3a1c997..68f4992 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,6 +8,7 @@ include("test_enumerable_keyed_aggregation.jl") include("test_enumerable_partitioning.jl") include("test_enumerable_combining.jl") include("test_enumerable_terminal.jl") +include("test_enumerable_typefiltering.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_typefiltering.jl b/test/test_enumerable_typefiltering.jl new file mode 100644 index 0000000..04a52dc --- /dev/null +++ b/test/test_enumerable_typefiltering.jl @@ -0,0 +1,89 @@ +@testitem "of_type" begin + source = QueryOperators.query(Any[1, "a", 2, "b", 3.0]) + + ints = QueryOperators.of_type(source, Int) + + @test collect(ints) == [1, 2] + @test eltype(ints) == Int + + strings = QueryOperators.of_type(source, String) + @test collect(strings) == ["a", "b"] + @test eltype(strings) == String +end + +@testitem "of_type keeps subtypes" begin + source = QueryOperators.query(Any[1, 2.0, "a"]) + + numbers = QueryOperators.of_type(source, Number) + + @test collect(numbers) == [1, 2.0] + @test eltype(numbers) == Number +end + +@testitem "of_type over a Union element type" begin + source = QueryOperators.query(Union{Int,String}[1, "a", 2]) + + @test collect(QueryOperators.of_type(source, Int)) == [1, 2] +end + +@testitem "of_type matching nothing or everything" begin + source = QueryOperators.query(Any[1, 2, 3]) + + @test collect(QueryOperators.of_type(source, String)) == String[] + @test collect(QueryOperators.of_type(source, Int)) == [1, 2, 3] +end + +@testitem "of_type on an empty source" begin + source = QueryOperators.query(Any[]) + + @test collect(QueryOperators.of_type(source, Int)) == Int[] +end + +@testitem "cast" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.cast(source, Float64) + + @test collect(res) == [1.0, 2.0, 3.0] + @test eltype(res) == Float64 + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "cast widens an Any source to a concrete type" begin + source = QueryOperators.query(Any[1, 2, 3]) + + res = QueryOperators.cast(source, Int) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int +end + +@testitem "cast fails on an element that cannot be converted" begin + source = QueryOperators.query(Any[1, "a"]) + + res = QueryOperators.cast(source, Int) + + @test_throws MethodError collect(res) +end + +@testitem "cast fails on a lossy conversion" begin + source = QueryOperators.query([1.5]) + + @test_throws InexactError collect(QueryOperators.cast(source, Int)) +end + +@testitem "cast on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.cast(source, Float64)) == Float64[] +end + +@testitem "of_type and cast compose with other operators" begin + source = QueryOperators.query(Any[1, "a", 2, "b", 3]) + + res = QueryOperators.cast(QueryOperators.of_type(source, Int), Float64) + + @test collect(res) == [1.0, 2.0, 3.0] + @test eltype(res) == Float64 +end From 8c8783af20154983656446dc9a5e0e9b44994ed0 Mon Sep 17 00:00:00 2001 From: David Anthoff Date: Wed, 16 Sep 2026 22:19:12 -0700 Subject: [PATCH 9/9] Bump version to v1.2.0-DEV and add release notes Co-Authored-By: Claude Opus 5 --- NEWS.md | 10 ++++++++++ Project.toml | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 2586223..1058a5a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,13 @@ +# QueryOperators.jl v1.2.0 Release Notes +* Add the left_join, right_join and full_join operators +* Add the concat, union, union_by, except, except_by, intersect and intersect_by operators +* Add the take_while, drop_while, take_last and drop_last operators +* Add the order, order_descending, reverse, shuffle and index operators +* Add the append, prepend and zip operators +* Add the count_by, aggregate_by and chunk operators +* Add the of_type and cast operators +* Add the min_by, max_by, any, all, contains, sequence_equal, aggregate, first, last, single and element_at terminal operators + # QueryOperators.jl v1.1.0 Release Notes * Add the summarize operator for grouped and whole-table aggregation diff --git a/Project.toml b/Project.toml index 7d1bf77..fb5c725 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "QueryOperators" uuid = "2aef5ad7-51ca-5a8f-8e88-e75cf067b44b" -version = "1.1.1-DEV" +version = "1.2.0-DEV" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"