diff --git a/NEWS.md b/NEWS.md index 2586223..1058a5a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,13 @@ +# QueryOperators.jl v1.2.0 Release Notes +* Add the left_join, right_join and full_join operators +* Add the concat, union, union_by, except, except_by, intersect and intersect_by operators +* Add the take_while, drop_while, take_last and drop_last operators +* Add the order, order_descending, reverse, shuffle and index operators +* Add the append, prepend and zip operators +* Add the count_by, aggregate_by and chunk operators +* Add the of_type and cast operators +* Add the min_by, max_by, any, all, contains, sequence_equal, aggregate, first, last, single and element_at terminal operators + # QueryOperators.jl v1.1.0 Release Notes * Add the summarize operator for grouped and whole-table aggregation diff --git a/Project.toml b/Project.toml index e1382fc..fb5c725 100644 --- a/Project.toml +++ b/Project.toml @@ -1,12 +1,13 @@ name = "QueryOperators" uuid = "2aef5ad7-51ca-5a8f-8e88-e75cf067b44b" -version = "1.1.1-DEV" +version = "1.2.0-DEV" [deps] DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" TableShowUtils = "5e66a065-1f0a-5976-b372-e0b8c017ca10" DataValues = "e7dc6d0d-1eca-5fa6-8ad6-5aecde8b7ea5" IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [extras] TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" @@ -18,6 +19,7 @@ DataStructures = "0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19" TableShowUtils = "0.1.1, 0.2, 0.3, 1" DataValues = "0.4.4, 0.5, 1" IteratorInterfaceExtensions = "0.1.1, 1" +Random = "1" [targets] test = ["Test", "TestItemRunner"] diff --git a/src/QueryOperators.jl b/src/QueryOperators.jl index 82c2379..ca610e7 100644 --- a/src/QueryOperators.jl +++ b/src/QueryOperators.jl @@ -4,6 +4,7 @@ using DataStructures using IteratorInterfaceExtensions using TableShowUtils import DataValues +import Random export Grouping, key @@ -14,6 +15,9 @@ include("enumerable/enumerable.jl") include("enumerable/enumerable_groupby.jl") include("enumerable/enumerable_join.jl") include("enumerable/enumerable_groupjoin.jl") +include("enumerable/enumerable_leftjoin.jl") +include("enumerable/enumerable_rightjoin.jl") +include("enumerable/enumerable_fulljoin.jl") include("enumerable/enumerable_orderby.jl") include("enumerable/enumerable_map.jl") include("enumerable/enumerable_filter.jl") @@ -22,9 +26,28 @@ include("enumerable/enumerable_defaultifempty.jl") include("enumerable/enumerable_count.jl") include("enumerable/enumerable_take.jl") include("enumerable/enumerable_drop.jl") +include("enumerable/enumerable_takewhile.jl") +include("enumerable/enumerable_dropwhile.jl") +include("enumerable/enumerable_takelast.jl") +include("enumerable/enumerable_droplast.jl") include("enumerable/enumerable_unique.jl") +include("enumerable/enumerable_concat.jl") +include("enumerable/enumerable_union.jl") +include("enumerable/enumerable_except.jl") +include("enumerable/enumerable_intersect.jl") +include("enumerable/enumerable_reverse.jl") +include("enumerable/enumerable_shuffle.jl") +include("enumerable/enumerable_index.jl") +include("enumerable/enumerable_append.jl") +include("enumerable/enumerable_prepend.jl") +include("enumerable/enumerable_zip.jl") +include("enumerable/enumerable_oftype.jl") include("enumerable/enumerable_pivot.jl") include("enumerable/enumerable_summarize.jl") +include("enumerable/enumerable_countby.jl") +include("enumerable/enumerable_aggregateby.jl") +include("enumerable/enumerable_chunk.jl") +include("enumerable/enumerable_terminal.jl") include("enumerable/show.jl") include("source_iterable.jl") diff --git a/src/enumerable/enumerable.jl b/src/enumerable/enumerable.jl index c6469a4..28914a1 100644 --- a/src/enumerable/enumerable.jl +++ b/src/enumerable/enumerable.jl @@ -4,3 +4,26 @@ Base.IteratorSize(::Type{T}) where {T <: Enumerable} = Base.SizeUnknown() IteratorInterfaceExtensions.isiterable(x::Enumerable) = true haslength(S) = Base.IteratorSize(S) isa Union{Base.HasLength, Base.HasShape} ? Base.HasLength() : Base.IteratorSize(S) + +# Operators that walk more than one source keep "which source, how far into it" +# in their iteration state. `_NotStarted` marks a source that has not been +# iterated yet, so that `_iterate_from` can pick the right `iterate` method +# without conflating it with a source whose own state happens to be `nothing`. +struct _NotStarted end + +_iterate_from(source, ::_NotStarted) = iterate(source) +_iterate_from(source, state) = iterate(source, state) + +# Element types of two sources that will be emitted into a single stream have +# to agree, the same requirement `default_if_empty` places on its default value. +function _check_same_eltype(op, ::Type{T1}, ::Type{T2}) where {T1,T2} + if T1 != T2 + error("The two sequences passed to $op have different element types, $T1 and $T2.") + end +end + +function _check_same_keytype(op, ::Type{TKey1}, ::Type{TKey2}) where {TKey1,TKey2} + if TKey1 != TKey2 + error("The keys of the two sequences passed to $op have different types, $TKey1 and $TKey2.") + end +end diff --git a/src/enumerable/enumerable_aggregateby.jl b/src/enumerable/enumerable_aggregateby.jl new file mode 100644 index 0000000..b185365 --- /dev/null +++ b/src/enumerable/enumerable_aggregateby.jl @@ -0,0 +1,49 @@ +_aggregate_by_row(k, v) = Base.merge(_key_namedtuple(k), (value = v,)) + +struct EnumerableAggregateBy{T,TKey,TACC,S,Q<:Function,A<:Function} <: Enumerable + source::S + f::Q + seed::TACC + accumulator::A +end + +Base.eltype(::Type{EnumerableAggregateBy{T,TKey,TACC,S,Q,A}}) where {T,TKey,TACC,S,Q,A} = T + +# Enumerable.AggregateBy (.NET 9): fold the elements of each key into a single +# value, without materialising the intermediate groupings. `accumulator` is +# called as `accumulator(accumulated, element)`, matching .NET's argument order. +# +# `summarize` is the more general and more idiomatic way to aggregate here; +# `aggregate_by` exists for LINQ parity. +function aggregate_by(source::Enumerable, f::Function, f_expr::Expr, seed, accumulator::Function) + TS = eltype(source) + TKey = Base._return_type(f, Tuple{TS,}) + TACC = typeof(seed) + T = Base._return_type(_aggregate_by_row, Tuple{TKey,TACC}) + + return EnumerableAggregateBy{T,TKey,TACC,typeof(source),typeof(f),typeof(accumulator)}(source, f, seed, accumulator) +end + +function Base.iterate(iter::EnumerableAggregateBy{T,TKey,TACC,S,Q,A}) where {T,TKey,TACC,S,Q,A} + accumulated = OrderedDict{TKey,TACC}() + for i in iter.source + k = iter.f(i) + accumulated[k] = iter.accumulator(get(accumulated, k, iter.seed), i) + end + + rows = T[_aggregate_by_row(k, v) for (k, v) in accumulated] + + if length(rows)==0 + return nothing + end + + return rows[1], (rows, 2) +end + +function Base.iterate(iter::EnumerableAggregateBy{T,TKey,TACC,S,Q,A}, state) where {T,TKey,TACC,S,Q,A} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_append.jl b/src/enumerable/enumerable_append.jl new file mode 100644 index 0000000..de92291 --- /dev/null +++ b/src/enumerable/enumerable_append.jl @@ -0,0 +1,32 @@ +struct EnumerableAppend{T,S} <: Enumerable + source::S + element::T +end + +# Enumerable.Append: the source followed by one more element. The element is +# converted to the source's element type, so appending an Int to a sequence of +# Float64 works. +function append(source::Enumerable, element) + T = eltype(source) + return EnumerableAppend{T,typeof(source)}(source, convert(T, element)) +end + +Base.IteratorSize(::Type{EnumerableAppend{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableAppend{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableAppend) = length(iter.source) + 1 + +Base.iterate(iter::EnumerableAppend) = _append_next(iter, _NotStarted()) + +function Base.iterate(iter::EnumerableAppend, state) + # `state.done` marks the appended element as already handed out. + state.done && return nothing + return _append_next(iter, state.state) +end + +function _append_next(iter::EnumerableAppend, source_state) + ret = _iterate_from(iter.source, source_state) + ret === nothing && return iter.element, (done=true, state=source_state) + return ret[1], (done=false, state=ret[2]) +end diff --git a/src/enumerable/enumerable_chunk.jl b/src/enumerable/enumerable_chunk.jl new file mode 100644 index 0000000..caf7075 --- /dev/null +++ b/src/enumerable/enumerable_chunk.jl @@ -0,0 +1,42 @@ +struct EnumerableChunk{T,TI,S} <: Enumerable + source::S + n::Int +end + +# Enumerable.Chunk (.NET 6): split the source into batches of at most `n` +# elements. The final batch is shorter when the source does not divide evenly. +function chunk(source::Enumerable, n::Integer) + n < 1 && error("The chunk size must be at least 1, got $n.") + + TI = eltype(source) + T = Vector{TI} + + return EnumerableChunk{T,TI,typeof(source)}(source, Int(n)) +end + +Base.IteratorSize(::Type{EnumerableChunk{T,TI,S}}) where {T,TI,S} = haslength(S) + +Base.eltype(::Type{EnumerableChunk{T,TI,S}}) where {T,TI,S} = T + +Base.length(iter::EnumerableChunk) = cld(length(iter.source), iter.n) + +Base.iterate(iter::EnumerableChunk) = _chunk_next(iter, _NotStarted()) + +Base.iterate(iter::EnumerableChunk, state) = _chunk_next(iter, state) + +# Pulls at most `n` elements per call, so a chunked source is only walked as +# far as the batches actually consumed. +function _chunk_next(iter::EnumerableChunk{T,TI,S}, source_state) where {T,TI,S} + buffer = TI[] + + while length(buffer) < iter.n + ret = _iterate_from(iter.source, source_state) + ret === nothing && break + push!(buffer, ret[1]) + source_state = ret[2] + end + + length(buffer)==0 && return nothing + + return buffer, source_state +end diff --git a/src/enumerable/enumerable_concat.jl b/src/enumerable/enumerable_concat.jl new file mode 100644 index 0000000..a0b5823 --- /dev/null +++ b/src/enumerable/enumerable_concat.jl @@ -0,0 +1,44 @@ +struct EnumerableConcat{T,S1,S2} <: Enumerable + first::S1 + second::S2 +end + +function concat(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("concat", T1, T2) + + return EnumerableConcat{T1,typeof(first),typeof(second)}(first, second) +end + +Base.eltype(::Type{EnumerableConcat{T,S1,S2}}) where {T,S1,S2} = T + +function Base.IteratorSize(::Type{EnumerableConcat{T,S1,S2}}) where {T,S1,S2} + return haslength(S1) isa Base.HasLength && haslength(S2) isa Base.HasLength ? + Base.HasLength() : Base.SizeUnknown() +end + +Base.length(iter::EnumerableConcat) = length(iter.first) + length(iter.second) + +Base.iterate(iter::EnumerableConcat) = _concat_next(iter, 1, _NotStarted()) + +function Base.iterate(iter::EnumerableConcat, state) + return _concat_next(iter, state.side, state.state) +end + +function _concat_next(iter::EnumerableConcat, side, source_state) + if side == 1 + ret = _iterate_from(iter.first, source_state) + if ret !== nothing + return ret[1], (side=1, state=ret[2]) + end + # First source exhausted — fall through to the second. + side = 2 + source_state = _NotStarted() + end + + ret = _iterate_from(iter.second, source_state) + ret === nothing && return nothing + return ret[1], (side=2, state=ret[2]) +end diff --git a/src/enumerable/enumerable_countby.jl b/src/enumerable/enumerable_countby.jl new file mode 100644 index 0000000..3217aad --- /dev/null +++ b/src/enumerable/enumerable_countby.jl @@ -0,0 +1,44 @@ +# Key columns are named exactly as `summarize` names them: a scalar key becomes +# a column called `key`, a NamedTuple key contributes one column per field. +_count_by_row(k, n::Int) = Base.merge(_key_namedtuple(k), (count = n,)) + +struct EnumerableCountBy{T,TKey,S,Q<:Function} <: Enumerable + source::S + f::Q +end + +Base.eltype(::Type{EnumerableCountBy{T,TKey,S,Q}}) where {T,TKey,S,Q} = T + +# Enumerable.CountBy (.NET 9): the frequency of each key, without materialising +# the intermediate groupings that `groupby` would build. +function count_by(source::Enumerable, f::Function, f_expr::Expr) + TS = eltype(source) + TKey = Base._return_type(f, Tuple{TS,}) + T = Base._return_type(_count_by_row, Tuple{TKey,Int}) + + return EnumerableCountBy{T,TKey,typeof(source),typeof(f)}(source, f) +end + +function Base.iterate(iter::EnumerableCountBy{T,TKey,S,Q}) where {T,TKey,S,Q} + counts = OrderedDict{TKey,Int}() + for i in iter.source + k = iter.f(i) + counts[k] = get(counts, k, 0) + 1 + end + + rows = T[_count_by_row(k, n) for (k, n) in counts] + + if length(rows)==0 + return nothing + end + + return rows[1], (rows, 2) +end + +function Base.iterate(iter::EnumerableCountBy{T,TKey,S,Q}, state) where {T,TKey,S,Q} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_defaultifempty.jl b/src/enumerable/enumerable_defaultifempty.jl index 9a9c7c4..45d10fa 100644 --- a/src/enumerable/enumerable_defaultifempty.jl +++ b/src/enumerable/enumerable_defaultifempty.jl @@ -13,6 +13,14 @@ function _default_value_expr(::Type{T}) where {T<:NamedTuple} return :( NamedTuple{$(fieldnames(T))}( ($( (_default_value_expr(fieldtype(T,i)) for i in 1:length(fieldnames(T)))... ),)) ) end +# Runtime counterpart of `_default_value_expr`, used by the outer join operators +# to build the all-null element that an unmatched side contributes. Generated so +# that the NamedTuple case is constructed at compile time, exactly as +# `default_if_empty` below does. +@generated function _default_value(::Type{T}) where {T} + return _default_value_expr(T) +end + @generated function default_if_empty(source::S) where {S} T_source = eltype(source) diff --git a/src/enumerable/enumerable_droplast.jl b/src/enumerable/enumerable_droplast.jl new file mode 100644 index 0000000..90885ab --- /dev/null +++ b/src/enumerable/enumerable_droplast.jl @@ -0,0 +1,48 @@ +struct EnumerableDropLast{T,S} <: Enumerable + source::S + n::Int +end + +# Enumerable.SkipLast: everything but the trailing `n` elements. A count of +# zero or less leaves the source unchanged, as in .NET. +function drop_last(source::Enumerable, n::Integer) + T = eltype(source) + return EnumerableDropLast{T,typeof(source)}(source, max(Int(n), 0)) +end + +Base.IteratorSize(::Type{EnumerableDropLast{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableDropLast{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableDropLast) = max(length(iter.source) - iter.n, 0) + +# Stays `n` elements behind the source: an element is only emitted once `n` +# further elements have been read, which proves it is not one of the last `n`. +function Base.iterate(iter::EnumerableDropLast{T,S}) where {T,S} + buffer = T[] + source_state = _NotStarted() + + while length(buffer) < iter.n + ret = _iterate_from(iter.source, source_state) + # Fewer than n elements in total, so every one of them is dropped. + ret === nothing && return nothing + push!(buffer, ret[1]) + source_state = ret[2] + end + + return _drop_last_next(iter, buffer, source_state) +end + +function Base.iterate(iter::EnumerableDropLast, state) + return _drop_last_next(iter, state.buffer, state.state) +end + +function _drop_last_next(iter::EnumerableDropLast, buffer, source_state) + ret = _iterate_from(iter.source, source_state) + ret === nothing && return nothing + + push!(buffer, ret[1]) + element = popfirst!(buffer) + + return element, (buffer=buffer, state=ret[2]) +end diff --git a/src/enumerable/enumerable_dropwhile.jl b/src/enumerable/enumerable_dropwhile.jl new file mode 100644 index 0000000..d7a8897 --- /dev/null +++ b/src/enumerable/enumerable_dropwhile.jl @@ -0,0 +1,27 @@ +struct EnumerableDropWhile{T,S,Q<:Function} <: Enumerable + source::S + f::Q +end + +# Enumerable.SkipWhile: discards the leading run of elements satisfying the +# predicate, then yields everything that follows without testing it again. +function drop_while(source::Enumerable, f::Function, f_expr::Expr) + T = eltype(source) + return EnumerableDropWhile{T,typeof(source),typeof(f)}(source, f) +end + +Base.eltype(::Type{EnumerableDropWhile{T,S,Q}}) where {T,S,Q} = T + +function Base.iterate(iter::EnumerableDropWhile) + ret = iterate(iter.source) + + while ret !== nothing && iter.f(ret[1]) + ret = iterate(iter.source, ret[2]) + end + + ret === nothing && return nothing + + return ret[1], ret[2] +end + +Base.iterate(iter::EnumerableDropWhile, state) = iterate(iter.source, state) diff --git a/src/enumerable/enumerable_except.jl b/src/enumerable/enumerable_except.jl new file mode 100644 index 0000000..10f3728 --- /dev/null +++ b/src/enumerable/enumerable_except.jl @@ -0,0 +1,62 @@ +struct EnumerableExcept{T,TKEY,S1,S2,Q<:Function} <: Enumerable + first::S1 + second::S2 + f::Q +end + +Base.eltype(::Type{EnumerableExcept{T,TKEY,S1,S2,Q}}) where {T,TKEY,S1,S2,Q} = T + +function except(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("except", T1, T2) + + return EnumerableExcept{T1,T1,typeof(first),typeof(second),typeof(identity)}(first, second, identity) +end + +# Unlike Enumerable.ExceptBy, whose second argument is a sequence of keys, the +# key selector here is applied to both sequences. That keeps `except_by` +# consistent with `union_by`, matches the shape of the equivalent SQL +# (`WHERE k NOT IN (SELECT k FROM b)`), and is what a table-shaped second +# argument makes natural. +function except_by(first::Enumerable, second::Enumerable, f::Function, f_expr::Expr) + T1 = eltype(first) + T2 = eltype(second) + + TKEY1 = Base._return_type(f, Tuple{T1,}) + TKEY2 = Base._return_type(f, Tuple{T2,}) + + _check_same_keytype("except_by", TKEY1, TKEY2) + + return EnumerableExcept{T1,TKEY1,typeof(first),typeof(second),typeof(f)}(first, second, f) +end + +function Base.iterate(iter::EnumerableExcept{T,TKEY,S1,S2,Q}) where {T,TKEY,S1,S2,Q} + excluded = Set{TKEY}() + for i in iter.second + push!(excluded, iter.f(i)) + end + + return _except_next(iter, excluded, Set{TKEY}(), _NotStarted()) +end + +function Base.iterate(iter::EnumerableExcept, state) + return _except_next(iter, state.excluded, state.observed, state.state) +end + +# Yields the distinct elements of the first source whose key does not occur in +# the second, matching Enumerable.Except's de-duplicating behaviour. +function _except_next(iter::EnumerableExcept, excluded, observed, source_state) + while true + ret = _iterate_from(iter.first, source_state) + ret === nothing && return nothing + + element, source_state = ret + k = iter.f(element) + if !(k in excluded) && !(k in observed) + push!(observed, k) + return element, (excluded=excluded, observed=observed, state=source_state) + end + end +end diff --git a/src/enumerable/enumerable_fulljoin.jl b/src/enumerable/enumerable_fulljoin.jl new file mode 100644 index 0000000..8420e32 --- /dev/null +++ b/src/enumerable/enumerable_fulljoin.jl @@ -0,0 +1,80 @@ +struct EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS<:Function,IKS<:Function,RS<:Function} <: Enumerable + outer::SO + inner::SI + outerKeySelector::OKS + innerKeySelector::IKS + resultSelector::RS + default_outer::TOD + default_inner::TID +end + +Base.eltype(::Type{EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS,IKS,RS}}) where {T,TKey,TOD,TID,SO,SI,OKS,IKS,RS} = T + +function full_join(outer::Enumerable, inner::Enumerable, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + TO = eltype(outer) + TI = eltype(inner) + TKeyOuter = Base._return_type(f_outerKeySelector, Tuple{TO,}) + TKeyInner = Base._return_type(f_innerKeySelector, Tuple{TI,}) + + _check_join_key_types("full_join", TKeyOuter, TKeyInner) + + default_outer = _default_value(TO) + default_inner = _default_value(TI) + TOD = typeof(default_outer) + TID = typeof(default_inner) + + T = Base._return_type(f_resultSelector, Tuple{TOD,TID}) + + SO = typeof(outer) + SI = typeof(inner) + OKS = typeof(f_outerKeySelector) + IKS = typeof(f_innerKeySelector) + RS = typeof(f_resultSelector) + + return EnumerableFullJoin{T,TKeyOuter,TOD,TID,SO,SI,OKS,IKS,RS}(outer, inner, f_outerKeySelector, f_innerKeySelector, f_resultSelector, default_outer, default_inner) +end + +function Base.iterate(iter::EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS,IKS,RS}) where {T,TKey,TOD,TID,SO,SI,OKS,IKS,RS} + results = Array{T}(undef, 0) + + inner_dict = _outerjoin_lookup(iter.inner, iter.innerKeySelector, TKey, TID) + + # All outer elements first, in source order: matched pairs where a key + # matches, otherwise the outer element paired with an all-null inner. + matched_keys = Set{TKey}() + for i in iter.outer + outerKey = iter.outerKeySelector(i) + converted_i = convert(TOD, i) + if haskey(inner_dict, outerKey) + push!(matched_keys, outerKey) + for j in inner_dict[outerKey] + push!(results, iter.resultSelector(converted_i, j)) + end + else + push!(results, iter.resultSelector(converted_i, iter.default_inner)) + end + end + + # Then the inner elements whose key never appeared on the outer side, + # in inner source order, paired with an all-null outer. + for (innerKey, elements) in inner_dict + innerKey in matched_keys && continue + for j in elements + push!(results, iter.resultSelector(iter.default_outer, j)) + end + end + + if length(results)==0 + return nothing + end + + return results[1], (results, 2) +end + +function Base.iterate(iter::EnumerableFullJoin{T,TKey,TOD,TID,SO,SI,OKS,IKS,RS}, state) where {T,TKey,TOD,TID,SO,SI,OKS,IKS,RS} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_index.jl b/src/enumerable/enumerable_index.jl new file mode 100644 index 0000000..997dcf5 --- /dev/null +++ b/src/enumerable/enumerable_index.jl @@ -0,0 +1,33 @@ +struct EnumerableIndex{T,TI,S} <: Enumerable + source::S +end + +# Enumerable.Index (.NET 9) pairs each element with its position. .NET yields +# (Index, Item) tuples; the Julia equivalent is an (index, item) NamedTuple. +# Indices are 1-based, matching the rest of Julia rather than .NET. +function index(source::Enumerable) + TI = eltype(source) + T = NamedTuple{(:index, :item),Tuple{Int,TI}} + return EnumerableIndex{T,TI,typeof(source)}(source) +end + +Base.IteratorSize(::Type{EnumerableIndex{T,TI,S}}) where {T,TI,S} = haslength(S) + +Base.eltype(::Type{EnumerableIndex{T,TI,S}}) where {T,TI,S} = T + +Base.length(iter::EnumerableIndex) = length(iter.source) + +function Base.iterate(iter::EnumerableIndex{T,TI,S}) where {T,TI,S} + ret = iterate(iter.source) + ret === nothing && return nothing + + return T((1, ret[1])), (i=1, state=ret[2]) +end + +function Base.iterate(iter::EnumerableIndex{T,TI,S}, state) where {T,TI,S} + ret = iterate(iter.source, state.state) + ret === nothing && return nothing + + i = state.i + 1 + return T((i, ret[1])), (i=i, state=ret[2]) +end diff --git a/src/enumerable/enumerable_intersect.jl b/src/enumerable/enumerable_intersect.jl new file mode 100644 index 0000000..1368de1 --- /dev/null +++ b/src/enumerable/enumerable_intersect.jl @@ -0,0 +1,59 @@ +struct EnumerableIntersect{T,TKEY,S1,S2,Q<:Function} <: Enumerable + first::S1 + second::S2 + f::Q +end + +Base.eltype(::Type{EnumerableIntersect{T,TKEY,S1,S2,Q}}) where {T,TKEY,S1,S2,Q} = T + +function intersect(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("intersect", T1, T2) + + return EnumerableIntersect{T1,T1,typeof(first),typeof(second),typeof(identity)}(first, second, identity) +end + +# As with `except_by`, the key selector is applied to both sequences rather than +# the second being a bare sequence of keys as in Enumerable.IntersectBy. +function intersect_by(first::Enumerable, second::Enumerable, f::Function, f_expr::Expr) + T1 = eltype(first) + T2 = eltype(second) + + TKEY1 = Base._return_type(f, Tuple{T1,}) + TKEY2 = Base._return_type(f, Tuple{T2,}) + + _check_same_keytype("intersect_by", TKEY1, TKEY2) + + return EnumerableIntersect{T1,TKEY1,typeof(first),typeof(second),typeof(f)}(first, second, f) +end + +function Base.iterate(iter::EnumerableIntersect{T,TKEY,S1,S2,Q}) where {T,TKEY,S1,S2,Q} + required = Set{TKEY}() + for i in iter.second + push!(required, iter.f(i)) + end + + return _intersect_next(iter, required, Set{TKEY}(), _NotStarted()) +end + +function Base.iterate(iter::EnumerableIntersect, state) + return _intersect_next(iter, state.required, state.observed, state.state) +end + +# Yields the distinct elements of the first source whose key also occurs in the +# second, matching Enumerable.Intersect's de-duplicating behaviour. +function _intersect_next(iter::EnumerableIntersect, required, observed, source_state) + while true + ret = _iterate_from(iter.first, source_state) + ret === nothing && return nothing + + element, source_state = ret + k = iter.f(element) + if k in required && !(k in observed) + push!(observed, k) + return element, (required=required, observed=observed, state=source_state) + end + end +end diff --git a/src/enumerable/enumerable_leftjoin.jl b/src/enumerable/enumerable_leftjoin.jl new file mode 100644 index 0000000..2289ce1 --- /dev/null +++ b/src/enumerable/enumerable_leftjoin.jl @@ -0,0 +1,85 @@ +# Builds the lookup from join key to the list of elements on one side of an +# outer join. Elements are converted to `TD` — the all-null-capable version of +# their own type — so that matched and unmatched rows have a single element +# type, mirroring how `EnumerableDefaultIfEmpty` converts its source. +function _outerjoin_lookup(source, keySelector, ::Type{TKey}, ::Type{TD}) where {TKey,TD} + lookup = OrderedDict{TKey,Array{TD,1}}() + for i in source + key = keySelector(i) + if !haskey(lookup, key) + lookup[key] = Array{TD}(undef, 0) + end + push!(lookup[key], convert(TD, i)) + end + return lookup +end + +function _check_join_key_types(op, ::Type{TKeyOuter}, ::Type{TKeyInner}) where {TKeyOuter,TKeyInner} + if TKeyOuter != TKeyInner + error("The keys in the $op clause have different types, $TKeyOuter and $TKeyInner.") + end +end + +struct EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS<:Function,IKS<:Function,RS<:Function} <: Enumerable + outer::SO + inner::SI + outerKeySelector::OKS + innerKeySelector::IKS + resultSelector::RS + default_inner::TID +end + +Base.eltype(::Type{EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS,IKS,RS}}) where {T,TKey,TID,SO,SI,OKS,IKS,RS} = T + +function left_join(outer::Enumerable, inner::Enumerable, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + TO = eltype(outer) + TI = eltype(inner) + TKeyOuter = Base._return_type(f_outerKeySelector, Tuple{TO,}) + TKeyInner = Base._return_type(f_innerKeySelector, Tuple{TI,}) + + _check_join_key_types("left_join", TKeyOuter, TKeyInner) + + default_inner = _default_value(TI) + TID = typeof(default_inner) + + T = Base._return_type(f_resultSelector, Tuple{TO,TID}) + + SO = typeof(outer) + SI = typeof(inner) + OKS = typeof(f_outerKeySelector) + IKS = typeof(f_innerKeySelector) + RS = typeof(f_resultSelector) + + return EnumerableLeftJoin{T,TKeyOuter,TID,SO,SI,OKS,IKS,RS}(outer, inner, f_outerKeySelector, f_innerKeySelector, f_resultSelector, default_inner) +end + +function Base.iterate(iter::EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS,IKS,RS}) where {T,TKey,TID,SO,SI,OKS,IKS,RS} + results = Array{T}(undef, 0) + + inner_dict = _outerjoin_lookup(iter.inner, iter.innerKeySelector, TKey, TID) + + for i in iter.outer + outerKey = iter.outerKeySelector(i) + if haskey(inner_dict, outerKey) + for j in inner_dict[outerKey] + push!(results, iter.resultSelector(i, j)) + end + else + push!(results, iter.resultSelector(i, iter.default_inner)) + end + end + + if length(results)==0 + return nothing + end + + return results[1], (results, 2) +end + +function Base.iterate(iter::EnumerableLeftJoin{T,TKey,TID,SO,SI,OKS,IKS,RS}, state) where {T,TKey,TID,SO,SI,OKS,IKS,RS} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_oftype.jl b/src/enumerable/enumerable_oftype.jl new file mode 100644 index 0000000..e675e1f --- /dev/null +++ b/src/enumerable/enumerable_oftype.jl @@ -0,0 +1,55 @@ +struct EnumerableOfType{T,S} <: Enumerable + source::S +end + +# Enumerable.OfType: keeps only the elements that are instances of T, and +# narrows the element type to T. Useful for sources whose element type is Any +# or a Union; a no-op filter on a homogeneously typed table. +function of_type(source::Enumerable, ::Type{T}) where {T} + return EnumerableOfType{T,typeof(source)}(source) +end + +Base.eltype(::Type{EnumerableOfType{T,S}}) where {T,S} = T + +Base.iterate(iter::EnumerableOfType) = _of_type_next(iter, _NotStarted()) + +Base.iterate(iter::EnumerableOfType, state) = _of_type_next(iter, state) + +function _of_type_next(iter::EnumerableOfType{T,S}, source_state) where {T,S} + while true + ret = _iterate_from(iter.source, source_state) + ret === nothing && return nothing + + element, source_state = ret + element isa T && return element, source_state + end +end + +struct EnumerableCast{T,S} <: Enumerable + source::S +end + +# Enumerable.Cast. .NET's Cast is a type assertion; the Julia counterpart is a +# conversion, so `cast` runs every element through `convert` and fails the same +# way `convert` would on an element that cannot be represented as T. +function cast(source::Enumerable, ::Type{T}) where {T} + return EnumerableCast{T,typeof(source)}(source) +end + +Base.IteratorSize(::Type{EnumerableCast{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableCast{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableCast) = length(iter.source) + +function Base.iterate(iter::EnumerableCast{T,S}) where {T,S} + ret = iterate(iter.source) + ret === nothing && return nothing + return convert(T, ret[1]), ret[2] +end + +function Base.iterate(iter::EnumerableCast{T,S}, state) where {T,S} + ret = iterate(iter.source, state) + ret === nothing && return nothing + return convert(T, ret[1]), ret[2] +end diff --git a/src/enumerable/enumerable_orderby.jl b/src/enumerable/enumerable_orderby.jl index b815d7e..496400b 100644 --- a/src/enumerable/enumerable_orderby.jl +++ b/src/enumerable/enumerable_orderby.jl @@ -28,6 +28,13 @@ function orderby_descending(source::Enumerable, f::Function, f_expr::Expr) return EnumerableOrderby{T,typeof(source),KS,TKS}(source, f, true) end +# Enumerable.Order/OrderDescending (.NET 7): sort by the elements themselves. +# Expressed in terms of orderby so that `thenby` can still follow them — it +# walks back up the source chain looking for an EnumerableOrderby. +order(source::Enumerable) = orderby(source, identity, :(i -> i)) + +order_descending(source::Enumerable) = orderby_descending(source, identity, :(i -> i)) + function Base.iterate(iter::EnumerableOrderby{T,S,KS,TKS}) where {T,S,KS,TKS} rows = (Base.IteratorSize(typeof(iter)) isa Base.HasLength || Base.IteratorSize(typeof(iter)) isa Base.HasShape) ? length(iter) : 0 diff --git a/src/enumerable/enumerable_pivot.jl b/src/enumerable/enumerable_pivot.jl index c38e2eb..1832018 100644 --- a/src/enumerable/enumerable_pivot.jl +++ b/src/enumerable/enumerable_pivot.jl @@ -165,7 +165,8 @@ end include_ops = (:include_name, :include_position, :include_startswith, :include_endswith, :include_occursin, :include_all, :include_range, :include_range_idx) - has_positive = any(inst[1] ∈ include_ops for inst in instructions) + # Base.any, not the `any` query operator this module also defines. + has_positive = Base.any(inst[1] ∈ include_ops for inst in instructions) result = has_positive ? Symbol[] : copy(all_names) diff --git a/src/enumerable/enumerable_prepend.jl b/src/enumerable/enumerable_prepend.jl new file mode 100644 index 0000000..7a643e4 --- /dev/null +++ b/src/enumerable/enumerable_prepend.jl @@ -0,0 +1,25 @@ +struct EnumerablePrepend{T,S} <: Enumerable + source::S + element::T +end + +# Enumerable.Prepend: one element followed by the source. As with `append`, the +# element is converted to the source's element type. +function prepend(source::Enumerable, element) + T = eltype(source) + return EnumerablePrepend{T,typeof(source)}(source, convert(T, element)) +end + +Base.IteratorSize(::Type{EnumerablePrepend{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerablePrepend{T,S}}) where {T,S} = T + +Base.length(iter::EnumerablePrepend) = length(iter.source) + 1 + +Base.iterate(iter::EnumerablePrepend) = (iter.element, (state=_NotStarted(),)) + +function Base.iterate(iter::EnumerablePrepend, state) + ret = _iterate_from(iter.source, state.state) + ret === nothing && return nothing + return ret[1], (state=ret[2],) +end diff --git a/src/enumerable/enumerable_reverse.jl b/src/enumerable/enumerable_reverse.jl new file mode 100644 index 0000000..bf4e681 --- /dev/null +++ b/src/enumerable/enumerable_reverse.jl @@ -0,0 +1,34 @@ +struct EnumerableReverse{T,S} <: Enumerable + source::S +end + +function reverse(source::Enumerable) + T = eltype(source) + return EnumerableReverse{T,typeof(source)}(source) +end + +Base.IteratorSize(::Type{EnumerableReverse{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableReverse{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableReverse) = length(iter.source) + +# Reversing needs the whole source, so the elements are collected on the first +# call and then handed out back to front. +function Base.iterate(iter::EnumerableReverse{T,S}) where {T,S} + elements = Base.collect(T, iter.source) + + if length(elements)==0 + return nothing + end + + return elements[end], (elements, length(elements)-1) +end + +function Base.iterate(iter::EnumerableReverse{T,S}, state) where {T,S} + if state[2]<1 + return nothing + else + return state[1][state[2]], (state[1], state[2]-1) + end +end diff --git a/src/enumerable/enumerable_rightjoin.jl b/src/enumerable/enumerable_rightjoin.jl new file mode 100644 index 0000000..f5ffd75 --- /dev/null +++ b/src/enumerable/enumerable_rightjoin.jl @@ -0,0 +1,63 @@ +struct EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS<:Function,IKS<:Function,RS<:Function} <: Enumerable + outer::SO + inner::SI + outerKeySelector::OKS + innerKeySelector::IKS + resultSelector::RS + default_outer::TOD +end + +Base.eltype(::Type{EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS,IKS,RS}}) where {T,TKey,TOD,SO,SI,OKS,IKS,RS} = T + +function right_join(outer::Enumerable, inner::Enumerable, f_outerKeySelector::Function, outerKeySelector::Expr, f_innerKeySelector::Function, innerKeySelector::Expr, f_resultSelector::Function, resultSelector::Expr) + TO = eltype(outer) + TI = eltype(inner) + TKeyOuter = Base._return_type(f_outerKeySelector, Tuple{TO,}) + TKeyInner = Base._return_type(f_innerKeySelector, Tuple{TI,}) + + _check_join_key_types("right_join", TKeyOuter, TKeyInner) + + default_outer = _default_value(TO) + TOD = typeof(default_outer) + + T = Base._return_type(f_resultSelector, Tuple{TOD,TI}) + + SO = typeof(outer) + SI = typeof(inner) + OKS = typeof(f_outerKeySelector) + IKS = typeof(f_innerKeySelector) + RS = typeof(f_resultSelector) + + return EnumerableRightJoin{T,TKeyInner,TOD,SO,SI,OKS,IKS,RS}(outer, inner, f_outerKeySelector, f_innerKeySelector, f_resultSelector, default_outer) +end + +function Base.iterate(iter::EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS,IKS,RS}) where {T,TKey,TOD,SO,SI,OKS,IKS,RS} + results = Array{T}(undef, 0) + + outer_dict = _outerjoin_lookup(iter.outer, iter.outerKeySelector, TKey, TOD) + + for j in iter.inner + innerKey = iter.innerKeySelector(j) + if haskey(outer_dict, innerKey) + for i in outer_dict[innerKey] + push!(results, iter.resultSelector(i, j)) + end + else + push!(results, iter.resultSelector(iter.default_outer, j)) + end + end + + if length(results)==0 + return nothing + end + + return results[1], (results, 2) +end + +function Base.iterate(iter::EnumerableRightJoin{T,TKey,TOD,SO,SI,OKS,IKS,RS}, state) where {T,TKey,TOD,SO,SI,OKS,IKS,RS} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_shuffle.jl b/src/enumerable/enumerable_shuffle.jl new file mode 100644 index 0000000..2b32a31 --- /dev/null +++ b/src/enumerable/enumerable_shuffle.jl @@ -0,0 +1,41 @@ +struct EnumerableShuffle{T,S,R} <: Enumerable + source::S + rng::R +end + +function shuffle(source::Enumerable) + return shuffle(source, Random.default_rng()) +end + +function shuffle(source::Enumerable, rng::Random.AbstractRNG) + T = eltype(source) + return EnumerableShuffle{T,typeof(source),typeof(rng)}(source, rng) +end + +Base.IteratorSize(::Type{EnumerableShuffle{T,S,R}}) where {T,S,R} = haslength(S) + +Base.eltype(::Type{EnumerableShuffle{T,S,R}}) where {T,S,R} = T + +Base.length(iter::EnumerableShuffle) = length(iter.source) + +# Like Enumerable.Shuffle, the randomisation is not cryptographically secure. +# Pass an explicit RNG to make a shuffle reproducible. +function Base.iterate(iter::EnumerableShuffle{T,S,R}) where {T,S,R} + elements = Base.collect(T, iter.source) + + if length(elements)==0 + return nothing + end + + Random.shuffle!(iter.rng, elements) + + return elements[1], (elements, 2) +end + +function Base.iterate(iter::EnumerableShuffle{T,S,R}, state) where {T,S,R} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_takelast.jl b/src/enumerable/enumerable_takelast.jl new file mode 100644 index 0000000..6ff0c98 --- /dev/null +++ b/src/enumerable/enumerable_takelast.jl @@ -0,0 +1,42 @@ +struct EnumerableTakeLast{T,S} <: Enumerable + source::S + n::Int +end + +# Enumerable.TakeLast: the trailing `n` elements. A count of zero or less +# yields nothing, as in .NET. +function take_last(source::Enumerable, n::Integer) + T = eltype(source) + return EnumerableTakeLast{T,typeof(source)}(source, max(Int(n), 0)) +end + +Base.IteratorSize(::Type{EnumerableTakeLast{T,S}}) where {T,S} = haslength(S) + +Base.eltype(::Type{EnumerableTakeLast{T,S}}) where {T,S} = T + +Base.length(iter::EnumerableTakeLast) = min(length(iter.source), iter.n) + +# Which elements are last is only known once the source is exhausted, so the +# whole source is walked, holding at most `n` elements in a ring buffer. +function Base.iterate(iter::EnumerableTakeLast{T,S}) where {T,S} + iter.n == 0 && return nothing + + buffer = CircularBuffer{T}(iter.n) + for i in iter.source + push!(buffer, i) + end + + length(buffer)==0 && return nothing + + elements = Base.collect(buffer) + + return elements[1], (elements, 2) +end + +function Base.iterate(iter::EnumerableTakeLast{T,S}, state) where {T,S} + if state[2]>length(state[1]) + return nothing + else + return state[1][state[2]], (state[1], state[2]+1) + end +end diff --git a/src/enumerable/enumerable_takewhile.jl b/src/enumerable/enumerable_takewhile.jl new file mode 100644 index 0000000..59cfe43 --- /dev/null +++ b/src/enumerable/enumerable_takewhile.jl @@ -0,0 +1,23 @@ +struct EnumerableTakeWhile{T,S,Q<:Function} <: Enumerable + source::S + f::Q +end + +# Enumerable.TakeWhile: yields elements until the predicate first fails, then +# stops — later elements are not examined even if they would satisfy it. +function take_while(source::Enumerable, f::Function, f_expr::Expr) + T = eltype(source) + return EnumerableTakeWhile{T,typeof(source),typeof(f)}(source, f) +end + +Base.eltype(::Type{EnumerableTakeWhile{T,S,Q}}) where {T,S,Q} = T + +Base.iterate(iter::EnumerableTakeWhile) = _take_while_step(iter, iterate(iter.source)) + +Base.iterate(iter::EnumerableTakeWhile, state) = _take_while_step(iter, iterate(iter.source, state)) + +function _take_while_step(iter::EnumerableTakeWhile, ret) + ret === nothing && return nothing + iter.f(ret[1]) || return nothing + return ret[1], ret[2] +end diff --git a/src/enumerable/enumerable_terminal.jl b/src/enumerable/enumerable_terminal.jl new file mode 100644 index 0000000..1ea702b --- /dev/null +++ b/src/enumerable/enumerable_terminal.jl @@ -0,0 +1,193 @@ +# Terminal operators. Unlike every other operator here these return a value +# rather than another Enumerable, so a query that ends in one is executed +# immediately. `count` in enumerable_count.jl is the pre-existing member of +# this family. + +_no_elements(op) = error("$op was called on a sequence with no elements.") + +_no_match(op) = error("$op was called on a sequence with no element matching the predicate.") + +# --- Quantifiers --- + +any(source::Enumerable) = iterate(source) !== nothing + +function any(source::Enumerable, f::Function, f_expr::Expr) + for i in source + f(i) && return true + end + return false +end + +function all(source::Enumerable, f::Function, f_expr::Expr) + for i in source + f(i) || return false + end + return true +end + +# Compares with isequal rather than ==, so that DataValue nulls match each +# other and NaN matches NaN, consistent with how `unique` and the set +# operators identify elements. +function contains(source::Enumerable, value) + for i in source + isequal(i, value) && return true + end + return false +end + +function sequence_equal(a::Enumerable, b::Enumerable) + sa = iterate(a) + sb = iterate(b) + + while sa !== nothing && sb !== nothing + isequal(sa[1], sb[1]) || return false + sa = iterate(a, sa[2]) + sb = iterate(b, sb[2]) + end + + # Equal only if both ran out at the same point. + return sa === nothing && sb === nothing +end + +# --- Extremes --- + +# Enumerable.MinBy/MaxBy (.NET 6) return the element itself, not the key. Ties +# keep the first such element, as in .NET, because the comparison is strict. +min_by(source::Enumerable, f::Function, f_expr::Expr) = + _extreme_by(source, f, isless, "min_by") + +max_by(source::Enumerable, f::Function, f_expr::Expr) = + _extreme_by(source, f, (candidate, best) -> isless(best, candidate), "max_by") + +function _extreme_by(source::Enumerable, f::Function, better::Function, op) + ret = iterate(source) + ret === nothing && _no_elements(op) + + best = ret[1] + best_key = f(best) + + ret = iterate(source, ret[2]) + while ret !== nothing + candidate_key = f(ret[1]) + if better(candidate_key, best_key) + best = ret[1] + best_key = candidate_key + end + ret = iterate(source, ret[2]) + end + + return best +end + +# --- Folds --- + +# Enumerable.Aggregate without a seed folds from the first element and requires +# a non-empty sequence. `summarize` is the idiomatic way to aggregate a table. +function aggregate(source::Enumerable, f::Function, f_expr::Expr) + ret = iterate(source) + ret === nothing && _no_elements("aggregate") + + accumulated = ret[1] + ret = iterate(source, ret[2]) + while ret !== nothing + accumulated = f(accumulated, ret[1]) + ret = iterate(source, ret[2]) + end + + return accumulated +end + +function aggregate(source::Enumerable, seed, f::Function, f_expr::Expr) + accumulated = seed + for i in source + accumulated = f(accumulated, i) + end + return accumulated +end + +# --- Element access --- + +function first(source::Enumerable) + ret = iterate(source) + ret === nothing && _no_elements("first") + return ret[1] +end + +function first(source::Enumerable, f::Function, f_expr::Expr) + for i in source + f(i) && return i + end + _no_match("first") +end + +function last(source::Enumerable) + ret = iterate(source) + ret === nothing && _no_elements("last") + + element = ret[1] + ret = iterate(source, ret[2]) + while ret !== nothing + element = ret[1] + ret = iterate(source, ret[2]) + end + + return element +end + +function last(source::Enumerable, f::Function, f_expr::Expr) + element = Base.Ref{eltype(source)}() + found = false + + for i in source + if f(i) + element[] = i + found = true + end + end + + found || _no_match("last") + + return element[] +end + +function single(source::Enumerable) + ret = iterate(source) + ret === nothing && _no_elements("single") + + element = ret[1] + iterate(source, ret[2]) === nothing || + error("single was called on a sequence with more than one element.") + + return element +end + +function single(source::Enumerable, f::Function, f_expr::Expr) + element = Base.Ref{eltype(source)}() + found = false + + for i in source + if f(i) + found && error("single was called on a sequence with more than one element matching the predicate.") + element[] = i + found = true + end + end + + found || _no_match("single") + + return element[] +end + +# 1-based, matching `index` and the rest of Julia rather than .NET's 0-based +# ElementAt. +function element_at(source::Enumerable, n::Integer) + n < 1 && error("element_at was called with index $n; the index must be at least 1.") + + seen = 0 + for i in source + seen += 1 + seen == n && return i + end + + error("element_at was called with index $n on a sequence with only $seen elements.") +end diff --git a/src/enumerable/enumerable_union.jl b/src/enumerable/enumerable_union.jl new file mode 100644 index 0000000..b884325 --- /dev/null +++ b/src/enumerable/enumerable_union.jl @@ -0,0 +1,58 @@ +struct EnumerableUnion{T,TKEY,S1,S2,Q<:Function} <: Enumerable + first::S1 + second::S2 + f::Q +end + +Base.eltype(::Type{EnumerableUnion{T,TKEY,S1,S2,Q}}) where {T,TKEY,S1,S2,Q} = T + +function union(first::Enumerable, second::Enumerable) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("union", T1, T2) + + return EnumerableUnion{T1,T1,typeof(first),typeof(second),typeof(identity)}(first, second, identity) +end + +function union_by(first::Enumerable, second::Enumerable, f::Function, f_expr::Expr) + T1 = eltype(first) + T2 = eltype(second) + + _check_same_eltype("union_by", T1, T2) + + TKEY = Base._return_type(f, Tuple{T1,}) + + return EnumerableUnion{T1,TKEY,typeof(first),typeof(second),typeof(f)}(first, second, f) +end + +function Base.iterate(iter::EnumerableUnion{T,TKEY,S1,S2,Q}) where {T,TKEY,S1,S2,Q} + return _union_next(iter, Set{TKEY}(), 1, _NotStarted()) +end + +function Base.iterate(iter::EnumerableUnion, state) + return _union_next(iter, state.observed, state.side, state.state) +end + +# Walks the first source and then the second, yielding each element whose key +# has not been seen before, so that the result is distinct across both sources. +function _union_next(iter::EnumerableUnion{T,TKEY,S1,S2,Q}, observed, side, source_state) where {T,TKEY,S1,S2,Q} + while true + source = side == 1 ? iter.first : iter.second + ret = _iterate_from(source, source_state) + + if ret === nothing + side == 2 && return nothing + side = 2 + source_state = _NotStarted() + continue + end + + element, source_state = ret + k = iter.f(element) + if !(k in observed) + push!(observed, k) + return element, (observed=observed, side=side, state=source_state) + end + end +end diff --git a/src/enumerable/enumerable_zip.jl b/src/enumerable/enumerable_zip.jl new file mode 100644 index 0000000..e80dd58 --- /dev/null +++ b/src/enumerable/enumerable_zip.jl @@ -0,0 +1,46 @@ +struct EnumerableZip{T,S1,S2,RS<:Function} <: Enumerable + first::S1 + second::S2 + resultSelector::RS +end + +Base.eltype(::Type{EnumerableZip{T,S1,S2,RS}}) where {T,S1,S2,RS} = T + +# Enumerable.Zip. Like Base.zip and .NET, the result stops at the shorter of +# the two sources — nothing is padded, so no null values are manufactured. +function zip(first::Enumerable, second::Enumerable) + return _zip(first, second, tuple) +end + +function zip(first::Enumerable, second::Enumerable, f_resultSelector::Function, resultSelector::Expr) + return _zip(first, second, f_resultSelector) +end + +function _zip(first::Enumerable, second::Enumerable, f_resultSelector::Function) + T1 = eltype(first) + T2 = eltype(second) + T = Base._return_type(f_resultSelector, Tuple{T1,T2}) + + return EnumerableZip{T,typeof(first),typeof(second),typeof(f_resultSelector)}(first, second, f_resultSelector) +end + +function Base.IteratorSize(::Type{EnumerableZip{T,S1,S2,RS}}) where {T,S1,S2,RS} + return haslength(S1) isa Base.HasLength && haslength(S2) isa Base.HasLength ? + Base.HasLength() : Base.SizeUnknown() +end + +Base.length(iter::EnumerableZip) = min(length(iter.first), length(iter.second)) + +Base.iterate(iter::EnumerableZip) = _zip_next(iter, _NotStarted(), _NotStarted()) + +Base.iterate(iter::EnumerableZip, state) = _zip_next(iter, state.s1, state.s2) + +function _zip_next(iter::EnumerableZip, s1, s2) + r1 = _iterate_from(iter.first, s1) + r1 === nothing && return nothing + + r2 = _iterate_from(iter.second, s2) + r2 === nothing && return nothing + + return iter.resultSelector(r1[1], r2[1]), (s1=r1[2], s2=r2[2]) +end diff --git a/src/operators.jl b/src/operators.jl index 563d3bb..74a3f45 100644 --- a/src/operators.jl +++ b/src/operators.jl @@ -129,3 +129,105 @@ function pivot_longer end function pivot_wider end function summarize end + +# Outer joins, mirroring Enumerable.LeftJoin/RightJoin/FullJoin (.NET 11). The +# unmatched side is supplied as an all-null element built by `_default_value`. + +function left_join end + +function right_join end + +function full_join end + +# Set operations. `union`, `except` and `intersect` shadow their Base +# counterparts, as `map`, `filter`, `count`, `take`, `unique` and `join` +# already do in this module. + +function concat end + +function union end + +function union_by end + +function except end + +function except_by end + +function intersect end + +function intersect_by end + +# Ordering and row position. `order`/`order_descending` are Enumerable.Order and +# OrderDescending (.NET 7); `shuffle` is Enumerable.Shuffle (.NET 10); `index` +# is Enumerable.Index (.NET 9). + +function order end + +function order_descending end + +function reverse end + +function shuffle end + +function index end + +# Keyed aggregation and batching. `count_by` and `aggregate_by` are +# Enumerable.CountBy and AggregateBy (.NET 9); `chunk` is Enumerable.Chunk +# (.NET 6). + +function count_by end + +function aggregate_by end + +function chunk end + +# Partitioning. `drop_while` and `drop_last` are Enumerable.SkipWhile and +# SkipLast, named to match the existing `drop` rather than LINQ's `Skip`. + +function take_while end + +function drop_while end + +function take_last end + +function drop_last end + +# Combining sequences. `append` and `zip` shadow their Base counterparts. + +function append end + +function prepend end + +function zip end + +# Terminal operators, which return a value rather than another Enumerable. +# `count` above is the pre-existing member of this family. Several of these +# shadow Base functions of the same name. + +function min_by end + +function max_by end + +function any end + +function all end + +function contains end + +function sequence_equal end + +function aggregate end + +function first end + +function last end + +function single end + +function element_at end + +# Type filtering. + +function of_type end + +function cast end diff --git a/test/runtests.jl b/test/runtests.jl index d5bb042..68f4992 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,14 @@ using TestItemRunner include("test_core.jl") +include("test_enumerable_outerjoins.jl") +include("test_enumerable_setops.jl") +include("test_enumerable_ordering.jl") +include("test_enumerable_keyed_aggregation.jl") +include("test_enumerable_partitioning.jl") +include("test_enumerable_combining.jl") +include("test_enumerable_terminal.jl") +include("test_enumerable_typefiltering.jl") include("test_enumerable_unique.jl") include("test_enumerable_summarize.jl") include("test_namedtupleutilities.jl") diff --git a/test/test_enumerable_combining.jl b/test/test_enumerable_combining.jl new file mode 100644 index 0000000..48e56b4 --- /dev/null +++ b/test/test_enumerable_combining.jl @@ -0,0 +1,154 @@ +@testitem "append" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.append(source, 4) + + @test collect(res) == [1, 2, 3, 4] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 4 +end + +@testitem "append to an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.append(source, 1)) == [1] +end + +@testitem "append converts the element to the source element type" begin + source = QueryOperators.query([1.0, 2.0]) + + res = QueryOperators.append(source, 3) + + @test collect(res) == [1.0, 2.0, 3.0] + @test eltype(res) == Float64 +end + +@testitem "append rejects an element that cannot be converted" begin + source = QueryOperators.query([1, 2]) + + @test_throws MethodError QueryOperators.append(source, "three") +end + +@testitem "append of NamedTuple rows" begin + source = QueryOperators.query([(a=1, b="x")]) + + res = QueryOperators.append(source, (a=2, b="y")) + + @test collect(res) == [(a=1, b="x"), (a=2, b="y")] +end + +@testitem "prepend" begin + source = QueryOperators.query([2, 3]) + + res = QueryOperators.prepend(source, 1) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "prepend to an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.prepend(source, 1)) == [1] +end + +@testitem "append and prepend compose" begin + source = QueryOperators.query([2, 3]) + + res = QueryOperators.append(QueryOperators.prepend(source, 1), 4) + + @test collect(res) == [1, 2, 3, 4] + @test length(res) == 4 +end + +@testitem "zip" begin + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query(["a", "b", "c"]) + + res = QueryOperators.zip(a, b) + + @test collect(res) == [(1, "a"), (2, "b"), (3, "c")] + @test eltype(res) == Tuple{Int,String} + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "zip truncates to the shorter source" begin + a = QueryOperators.query([1, 2, 3, 4]) + b = QueryOperators.query(["a", "b"]) + + @test collect(QueryOperators.zip(a, b)) == [(1, "a"), (2, "b")] + @test length(QueryOperators.zip(a, b)) == 2 + @test collect(QueryOperators.zip(b, a)) == [("a", 1), ("b", 2)] +end + +@testitem "zip with an empty source" begin + a = QueryOperators.query([1, 2]) + empty = QueryOperators.query(String[]) + + @test collect(QueryOperators.zip(a, empty)) == Tuple{Int,String}[] + @test collect(QueryOperators.zip(empty, a)) == Tuple{String,Int}[] +end + +@testitem "zip with a result selector" begin + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query([10, 20, 30]) + + res = QueryOperators.zip(a, b, (x, y) -> (sum=x + y,), :((x, y) -> (sum=x + y,))) + + @test collect(res) == [(sum=11,), (sum=22,), (sum=33,)] + @test eltype(res) == NamedTuple{(:sum,),Tuple{Int}} +end + +@testitem "zip pads nothing, so it manufactures no null values" begin + using DataValues + + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query([10]) + + res = collect(QueryOperators.zip(a, b)) + + @test length(res) == 1 + @test !any(r -> any(ismissing, r), res) + @test !any(r -> any(x -> x isa DataValue, r), res) +end + +@testitem "zip does not over-read the longer source" begin + import IteratorInterfaceExtensions + + mutable struct CountedZipSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedZipSource}) = Int + Base.IteratorSize(::Type{CountedZipSource}) = Base.HasLength() + Base.length(c::CountedZipSource) = length(c.data) + function Base.iterate(c::CountedZipSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedZipSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedZipSource) = c + + long = CountedZipSource(collect(1:100), 0) + res = QueryOperators.zip(QueryOperators.query(long), QueryOperators.query([1, 2])) + + @test length(collect(res)) == 2 + # Three pulls: two matched, plus the one that had no partner. + @test long.pulled == 3 +end + +@testitem "combining operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + zipped = collect(QueryOperators.zip(grouped, QueryOperators.query(["first", "second"]))) + + @test length(zipped) == 2 + @test [QueryOperators.key(g) for (g, _) in zipped] == [1, 2] + @test [s for (_, s) in zipped] == ["first", "second"] +end diff --git a/test/test_enumerable_keyed_aggregation.jl b/test/test_enumerable_keyed_aggregation.jl new file mode 100644 index 0000000..15d78ff --- /dev/null +++ b/test/test_enumerable_keyed_aggregation.jl @@ -0,0 +1,165 @@ +@testitem "count_by with a scalar key" begin + source = QueryOperators.query([(k="a", v=1), (k="b", v=2), (k="a", v=3)]) + + res = QueryOperators.count_by(source, i -> i.k, :(i -> i.k)) + + # A scalar key becomes a column called `key`, as summarize names it. + @test collect(res) == [(key="a", count=2), (key="b", count=1)] + @test eltype(res) == NamedTuple{(:key, :count),Tuple{String,Int}} +end + +@testitem "count_by with a NamedTuple key splats the key columns" begin + source = QueryOperators.query([(a=1, b=1), (a=1, b=1), (a=1, b=2)]) + + res = QueryOperators.count_by(source, i -> (a=i.a, b=i.b), :(i -> (a=i.a, b=i.b))) + + @test collect(res) == [(a=1, b=1, count=2), (a=1, b=2, count=1)] +end + +@testitem "count_by preserves first-seen key order" begin + source = QueryOperators.query([3, 1, 3, 2, 1]) + + res = QueryOperators.count_by(source, i -> i, :(i -> i)) + + @test collect(res) == [(key=3, count=2), (key=1, count=2), (key=2, count=1)] +end + +@testitem "count_by on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.count_by(source, i -> i, :(i -> i))) == NamedTuple{(:key, :count),Tuple{Int,Int}}[] +end + +@testitem "count_by agrees with groupby plus summarize" begin + using Statistics + + data = [(k=1, v=10), (k=2, v=20), (k=1, v=30), (k=3, v=40), (k=1, v=50)] + + by_count = collect(QueryOperators.count_by(QueryOperators.query(data), i -> i.k, :(i -> i.k))) + + grouped = QueryOperators.@groupby_simple(QueryOperators.query(data), i -> i.k) + by_summarize = collect(QueryOperators.summarize( + grouped, + g -> Base.merge(QueryOperators._key_namedtuple(QueryOperators.key(g)), (count = length(g),)), + :(g -> (count = length(g),)))) + + @test by_count == by_summarize +end + +@testitem "aggregate_by" begin + source = QueryOperators.query([(id="0", score=42), (id="1", score=5), (id="2", score=4), (id="1", score=10), (id="0", score=25)]) + + res = QueryOperators.aggregate_by( + source, + i -> i.id, :(i -> i.id), + 0, + (total, cur) -> total + cur.score) + + # The example from the .NET 9 release notes. + @test collect(res) == [(key="0", value=67), (key="1", value=15), (key="2", value=4)] + @test eltype(res) == NamedTuple{(:key, :value),Tuple{String,Int}} +end + +@testitem "aggregate_by with a NamedTuple key" begin + source = QueryOperators.query([(a=1, b=1, v=2), (a=1, b=1, v=3), (a=2, b=1, v=5)]) + + res = QueryOperators.aggregate_by( + source, + i -> (a=i.a, b=i.b), :(i -> (a=i.a, b=i.b)), + 1, + (acc, cur) -> acc * cur.v) + + @test collect(res) == [(a=1, b=1, value=6), (a=2, b=1, value=5)] +end + +@testitem "aggregate_by on an empty source" begin + source = QueryOperators.query(Int[]) + + res = QueryOperators.aggregate_by(source, i -> i, :(i -> i), 0, (acc, cur) -> acc + cur) + + @test collect(res) == NamedTuple{(:key, :value),Tuple{Int,Int}}[] +end + +@testitem "aggregate_by never shares the seed between keys" begin + source = QueryOperators.query([1, 1, 2, 2, 2]) + + res = QueryOperators.aggregate_by(source, i -> i, :(i -> i), Int[], (acc, cur) -> vcat(acc, cur)) + + @test collect(res) == [(key=1, value=[1, 1]), (key=2, value=[2, 2, 2])] +end + +@testitem "chunk" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + res = QueryOperators.chunk(source, 2) + + # The final chunk is short when the source does not divide evenly. + @test collect(res) == [[1, 2], [3, 4], [5]] + @test eltype(res) == Vector{Int} + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "chunk when the size divides evenly" begin + source = QueryOperators.query([1, 2, 3, 4]) + + @test collect(QueryOperators.chunk(source, 2)) == [[1, 2], [3, 4]] + @test length(QueryOperators.chunk(source, 2)) == 2 +end + +@testitem "chunk with a size larger than the source" begin + source = QueryOperators.query([1, 2]) + + @test collect(QueryOperators.chunk(source, 10)) == [[1, 2]] +end + +@testitem "chunk on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.chunk(source, 3)) == Vector{Int}[] +end + +@testitem "chunk rejects a size below 1" begin + source = QueryOperators.query([1, 2, 3]) + + @test_throws ErrorException QueryOperators.chunk(source, 0) + @test_throws ErrorException QueryOperators.chunk(source, -1) +end + +@testitem "chunk only walks the source as far as the batches consumed" begin + import IteratorInterfaceExtensions + + mutable struct CountedChunkSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedChunkSource}) = Int + Base.IteratorSize(::Type{CountedChunkSource}) = Base.HasLength() + Base.length(c::CountedChunkSource) = length(c.data) + function Base.iterate(c::CountedChunkSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedChunkSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedChunkSource) = c + + src = CountedChunkSource(collect(1:100), 0) + res = QueryOperators.chunk(QueryOperators.query(src), 3) + + it = iterate(res) + + @test it[1] == [1, 2, 3] + @test src.pulled == 3 +end + +@testitem "chunk works downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2), (k=3, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + chunks = collect(QueryOperators.chunk(grouped, 2)) + + @test length(chunks) == 2 + @test [QueryOperators.key(g) for g in chunks[1]] == [1, 2] + @test [QueryOperators.key(g) for g in chunks[2]] == [3] +end diff --git a/test/test_enumerable_ordering.jl b/test/test_enumerable_ordering.jl new file mode 100644 index 0000000..269fe7c --- /dev/null +++ b/test/test_enumerable_ordering.jl @@ -0,0 +1,141 @@ +@testitem "order" begin + source = QueryOperators.query([3, 1, 2]) + + res = QueryOperators.order(source) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "order_descending" begin + source = QueryOperators.query([3, 1, 2]) + + @test collect(QueryOperators.order_descending(source)) == [3, 2, 1] +end + +@testitem "order on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.order(source)) == Int[] + @test collect(QueryOperators.order_descending(source)) == Int[] +end + +@testitem "thenby can follow order" begin + source = QueryOperators.query([(a=1, b=2), (a=1, b=1), (a=0, b=9)]) + + # order sorts by the whole element; thenby then refines it, which only + # works because order reuses EnumerableOrderby. + res = QueryOperators.@thenby(QueryOperators.order(source), i -> i.b) + + @test collect(res) == [(a=0, b=9), (a=1, b=1), (a=1, b=2)] +end + +@testitem "reverse" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.reverse(source) + + @test collect(res) == [3, 2, 1] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "reverse on empty and single-element sources" begin + @test collect(QueryOperators.reverse(QueryOperators.query(Int[]))) == Int[] + @test collect(QueryOperators.reverse(QueryOperators.query([7]))) == [7] +end + +@testitem "reverse of reverse is the identity" begin + source = QueryOperators.query([1, 2, 3, 4]) + + @test collect(QueryOperators.reverse(QueryOperators.reverse(source))) == [1, 2, 3, 4] +end + +@testitem "shuffle is a permutation of its source" begin + using Random + + source = QueryOperators.query(collect(1:50)) + + res = QueryOperators.shuffle(source) + + @test sort(collect(res)) == collect(1:50) + @test eltype(res) == Int + @test length(res) == 50 +end + +@testitem "shuffle with an explicit rng is reproducible" begin + using Random + + source = QueryOperators.query(collect(1:50)) + + a = collect(QueryOperators.shuffle(source, MersenneTwister(42))) + b = collect(QueryOperators.shuffle(source, MersenneTwister(42))) + + @test a == b + @test sort(a) == collect(1:50) +end + +@testitem "shuffle on an empty source" begin + @test collect(QueryOperators.shuffle(QueryOperators.query(Int[]))) == Int[] +end + +@testitem "index" begin + source = QueryOperators.query(["a", "b", "c"]) + + res = QueryOperators.index(source) + + # 1-based, matching Julia rather than .NET's 0-based Index(). + @test collect(res) == [(index=1, item="a"), (index=2, item="b"), (index=3, item="c")] + @test eltype(res) == NamedTuple{(:index, :item),Tuple{Int,String}} + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "index on an empty source" begin + source = QueryOperators.query(String[]) + + @test collect(QueryOperators.index(source)) == NamedTuple{(:index, :item),Tuple{Int,String}}[] +end + +@testitem "index is lazy" begin + import IteratorInterfaceExtensions + + mutable struct CountedIndexSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedIndexSource}) = Int + Base.IteratorSize(::Type{CountedIndexSource}) = Base.HasLength() + Base.length(c::CountedIndexSource) = length(c.data) + function Base.iterate(c::CountedIndexSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedIndexSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedIndexSource) = c + + src = CountedIndexSource([1, 2, 3, 4, 5], 0) + res = QueryOperators.index(QueryOperators.query(src)) + + it = iterate(res) + it = iterate(res, it[2]) + + @test it[1] == (index=2, item=2) + @test src.pulled == 2 +end + +@testitem "ordering operators work downstream of groupby" begin + source = QueryOperators.query([(k=2, v=1), (k=1, v=2), (k=3, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + reversed = collect(QueryOperators.reverse(grouped)) + @test [QueryOperators.key(g) for g in reversed] == [3, 1, 2] + + indexed = collect(QueryOperators.index(grouped)) + @test [i.index for i in indexed] == [1, 2, 3] + @test [QueryOperators.key(i.item) for i in indexed] == [2, 1, 3] +end diff --git a/test/test_enumerable_outerjoins.jl b/test/test_enumerable_outerjoins.jl new file mode 100644 index 0000000..08f3a54 --- /dev/null +++ b/test/test_enumerable_outerjoins.jl @@ -0,0 +1,191 @@ +@testitem "left_join" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=3, x="c")]) + inner = QueryOperators.query([(a=1, y=10), (a=1, y=11), (a=3, y=30)]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 4 + @test res[1] == (x="a", y=DataValue(10)) + @test res[2] == (x="a", y=DataValue(11)) + @test res[3].x == "b" + @test isna(res[3].y) + @test res[4] == (x="c", y=DataValue(30)) + + # Absent values are DataValue, never missing. + @test eltype(res) == NamedTuple{(:x, :y),Tuple{String,DataValue{Int}}} + @test !any(r -> ismissing(r.y), res) +end + +@testitem "left_join keeps every outer row when the inner side is empty" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=2, x="b")]) + inner = QueryOperators.query(NamedTuple{(:a, :y),Tuple{Int,Int}}[]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 2 + @test all(r -> isna(r.y), res) +end + +@testitem "left_join on an empty outer side yields nothing" begin + outer = QueryOperators.query(NamedTuple{(:a, :x),Tuple{Int,String}}[]) + inner = QueryOperators.query([(a=1, y=10)]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 0 +end + +@testitem "left_join rejects mismatched key types" begin + outer = QueryOperators.query([(a=1, x="a")]) + inner = QueryOperators.query([(a="1", y=10)]) + + @test_throws ErrorException QueryOperators.left_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y))) +end + +@testitem "left_join over scalar elements" begin + using DataValues + + outer = QueryOperators.query([1, 2, 3]) + inner = QueryOperators.query([1, 3]) + + res = collect(QueryOperators.left_join( + outer, inner, + i -> i, :(i -> i), + i -> i, :(i -> i), + (i, j) -> (o=i, i=j), :((i, j) -> (o=i, i=j)))) + + @test length(res) == 3 + @test res[1] == (o=1, i=DataValue(1)) + @test isna(res[2].i) + @test res[3] == (o=3, i=DataValue(3)) +end + +@testitem "right_join" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=3, x="c")]) + inner = QueryOperators.query([(a=1, y=10), (a=2, y=20), (a=3, y=30)]) + + res = collect(QueryOperators.right_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + # One row per inner element, in inner order. + @test length(res) == 3 + @test res[1] == (x=DataValue("a"), y=10) + @test isna(res[2].x) + @test res[2].y == 20 + @test res[3] == (x=DataValue("c"), y=30) + + @test eltype(res) == NamedTuple{(:x, :y),Tuple{DataValue{String},Int}} + @test !any(r -> ismissing(r.x), res) +end + +@testitem "right_join keeps every inner row when the outer side is empty" begin + using DataValues + + outer = QueryOperators.query(NamedTuple{(:a, :x),Tuple{Int,String}}[]) + inner = QueryOperators.query([(a=1, y=10), (a=2, y=20)]) + + res = collect(QueryOperators.right_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 2 + @test all(r -> isna(r.x), res) +end + +@testitem "full_join" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=2, x="b")]) + inner = QueryOperators.query([(a=1, y=10), (a=3, y=30)]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + # Outer rows first in outer order, then unmatched inner rows. + @test length(res) == 3 + @test res[1] == (x=DataValue("a"), y=DataValue(10)) + @test res[2].x == DataValue("b") + @test isna(res[2].y) + @test isna(res[3].x) + @test res[3].y == DataValue(30) + + @test eltype(res) == NamedTuple{(:x, :y),Tuple{DataValue{String},DataValue{Int}}} + @test !any(r -> ismissing(r.x) || ismissing(r.y), res) +end + +@testitem "full_join with no overlap keeps both sides" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a")]) + inner = QueryOperators.query([(a=2, y=20)]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 2 + @test isna(res[1].y) + @test isna(res[2].x) +end + +@testitem "full_join with duplicate keys on both sides" begin + using DataValues + + outer = QueryOperators.query([(a=1, x="a"), (a=1, x="b")]) + inner = QueryOperators.query([(a=1, y=10), (a=1, y=11)]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + # Cartesian product within the matching key, as for an inner join. + @test length(res) == 4 + @test [(get(r.x), get(r.y)) for r in res] == [("a", 10), ("a", 11), ("b", 10), ("b", 11)] +end + +@testitem "full_join on two empty sides yields nothing" begin + outer = QueryOperators.query(NamedTuple{(:a, :x),Tuple{Int,String}}[]) + inner = QueryOperators.query(NamedTuple{(:a, :y),Tuple{Int,Int}}[]) + + res = collect(QueryOperators.full_join( + outer, inner, + i -> i.a, :(i -> i.a), + i -> i.a, :(i -> i.a), + (i, j) -> (x=i.x, y=j.y), :((i, j) -> (x=i.x, y=j.y)))) + + @test length(res) == 0 +end diff --git a/test/test_enumerable_partitioning.jl b/test/test_enumerable_partitioning.jl new file mode 100644 index 0000000..668b979 --- /dev/null +++ b/test/test_enumerable_partitioning.jl @@ -0,0 +1,165 @@ +@testitem "take_while" begin + source = QueryOperators.query([1, 2, 3, 4, 1, 2]) + + res = QueryOperators.take_while(source, i -> i < 3, :(i -> i < 3)) + + # Stops at the first failure; the trailing 1 and 2 are not taken. + @test collect(res) == [1, 2] + @test eltype(res) == Int +end + +@testitem "take_while taking everything or nothing" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.take_while(source, i -> true, :(i -> true))) == [1, 2, 3] + @test collect(QueryOperators.take_while(source, i -> false, :(i -> false))) == Int[] +end + +@testitem "take_while on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.take_while(source, i -> true, :(i -> true))) == Int[] +end + +@testitem "take_while stops walking the source at the first failure" begin + import IteratorInterfaceExtensions + + mutable struct CountedTakeWhileSource + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{CountedTakeWhileSource}) = Int + Base.IteratorSize(::Type{CountedTakeWhileSource}) = Base.HasLength() + Base.length(c::CountedTakeWhileSource) = length(c.data) + function Base.iterate(c::CountedTakeWhileSource, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::CountedTakeWhileSource) = true + IteratorInterfaceExtensions.getiterator(c::CountedTakeWhileSource) = c + + src = CountedTakeWhileSource(collect(1:100), 0) + res = QueryOperators.take_while(QueryOperators.query(src), i -> i < 4, :(i -> i < 4)) + + @test collect(res) == [1, 2, 3] + # Three taken plus the one that failed the predicate. + @test src.pulled == 4 +end + +@testitem "drop_while" begin + source = QueryOperators.query([1, 2, 3, 4, 1, 2]) + + res = QueryOperators.drop_while(source, i -> i < 3, :(i -> i < 3)) + + # Only the leading run is dropped; the trailing 1 and 2 survive. + @test collect(res) == [3, 4, 1, 2] + @test eltype(res) == Int +end + +@testitem "drop_while dropping everything or nothing" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.drop_while(source, i -> true, :(i -> true))) == Int[] + @test collect(QueryOperators.drop_while(source, i -> false, :(i -> false))) == [1, 2, 3] +end + +@testitem "drop_while on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.drop_while(source, i -> true, :(i -> true))) == Int[] +end + +@testitem "take_while and drop_while partition the source" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + p = i -> i < 3 + + taken = collect(QueryOperators.take_while(source, p, :(i -> i < 3))) + dropped = collect(QueryOperators.drop_while(source, p, :(i -> i < 3))) + + @test vcat(taken, dropped) == [1, 2, 3, 4, 5] +end + +@testitem "take_last" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + res = QueryOperators.take_last(source, 2) + + @test collect(res) == [4, 5] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 2 +end + +@testitem "take_last with a count at or beyond the source length" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.take_last(source, 3)) == [1, 2, 3] + @test collect(QueryOperators.take_last(source, 10)) == [1, 2, 3] + @test length(QueryOperators.take_last(source, 10)) == 3 +end + +@testitem "take_last with a count of zero or less" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.take_last(source, 0)) == Int[] + @test collect(QueryOperators.take_last(source, -1)) == Int[] +end + +@testitem "take_last on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.take_last(source, 2)) == Int[] +end + +@testitem "drop_last" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + res = QueryOperators.drop_last(source, 2) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "drop_last with a count at or beyond the source length" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.drop_last(source, 3)) == Int[] + @test collect(QueryOperators.drop_last(source, 10)) == Int[] + @test length(QueryOperators.drop_last(source, 10)) == 0 +end + +@testitem "drop_last with a count of zero or less is the identity" begin + source = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.drop_last(source, 0)) == [1, 2, 3] + @test collect(QueryOperators.drop_last(source, -1)) == [1, 2, 3] +end + +@testitem "drop_last on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.drop_last(source, 2)) == Int[] +end + +@testitem "take_last and drop_last partition the source" begin + source = QueryOperators.query([1, 2, 3, 4, 5]) + + dropped = collect(QueryOperators.drop_last(source, 2)) + taken = collect(QueryOperators.take_last(source, 2)) + + @test vcat(dropped, taken) == [1, 2, 3, 4, 5] +end + +@testitem "partitioning operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2), (k=3, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + last_two = collect(QueryOperators.take_last(grouped, 2)) + @test [QueryOperators.key(g) for g in last_two] == [2, 3] + + all_but_last = collect(QueryOperators.drop_last(grouped, 1)) + @test [QueryOperators.key(g) for g in all_but_last] == [1, 2] +end diff --git a/test/test_enumerable_setops.jl b/test/test_enumerable_setops.jl new file mode 100644 index 0000000..d6616a8 --- /dev/null +++ b/test/test_enumerable_setops.jl @@ -0,0 +1,188 @@ +@testitem "concat" begin + a = QueryOperators.query([1, 2, 3]) + b = QueryOperators.query([3, 4]) + + res = QueryOperators.concat(a, b) + + # Concat keeps duplicates and source order. + @test collect(res) == [1, 2, 3, 3, 4] + @test eltype(res) == Int + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 5 +end + +@testitem "concat with an empty side" begin + a = QueryOperators.query([1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.concat(a, empty)) == [1, 2] + @test collect(QueryOperators.concat(empty, a)) == [1, 2] + @test collect(QueryOperators.concat(empty, empty)) == Int[] +end + +@testitem "concat rejects mismatched element types" begin + a = QueryOperators.query([1, 2]) + b = QueryOperators.query(["a"]) + + @test_throws ErrorException QueryOperators.concat(a, b) +end + +@testitem "concat does not touch the second source until the first is exhausted" begin + import IteratorInterfaceExtensions + + # A source that records how many elements have been pulled from it. + mutable struct Counted + data::Vector{Int} + pulled::Int + end + Base.eltype(::Type{Counted}) = Int + Base.IteratorSize(::Type{Counted}) = Base.HasLength() + Base.length(c::Counted) = length(c.data) + function Base.iterate(c::Counted, i=1) + i > length(c.data) && return nothing + c.pulled += 1 + return c.data[i], i + 1 + end + IteratorInterfaceExtensions.isiterable(::Counted) = true + IteratorInterfaceExtensions.getiterator(c::Counted) = c + + second = Counted([10, 20], 0) + res = QueryOperators.concat(QueryOperators.query([1, 2, 3]), QueryOperators.query(second)) + + # Pull two of the three elements of the first source. + it = iterate(res) + it = iterate(res, it[2]) + + @test it[1] == 2 + @test second.pulled == 0 + + # Draining the rest reaches the second source. + @test collect(res) == [1, 2, 3, 10, 20] + @test second.pulled > 0 +end + +@testitem "union" begin + a = QueryOperators.query([1, 2, 2, 3]) + b = QueryOperators.query([3, 4, 4]) + + res = QueryOperators.union(a, b) + + # Distinct across both sources, in first-seen order. + @test collect(res) == [1, 2, 3, 4] + @test eltype(res) == Int +end + +@testitem "union with an empty side" begin + a = QueryOperators.query([1, 1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.union(a, empty)) == [1, 2] + @test collect(QueryOperators.union(empty, a)) == [1, 2] + @test collect(QueryOperators.union(empty, empty)) == Int[] +end + +@testitem "union_by" begin + a = QueryOperators.query([(a=1, x="a"), (a=2, x="b")]) + b = QueryOperators.query([(a=2, x="B"), (a=3, x="c")]) + + res = QueryOperators.union_by(a, b, i -> i.a, :(i -> i.a)) + + # The first element seen for a key wins, so a=2 keeps x="b". + @test collect(res) == [(a=1, x="a"), (a=2, x="b"), (a=3, x="c")] +end + +@testitem "except" begin + a = QueryOperators.query([1, 2, 2, 3, 4]) + b = QueryOperators.query([2, 4]) + + res = QueryOperators.except(a, b) + + # Distinct elements of the first source not present in the second. + @test collect(res) == [1, 3] + @test eltype(res) == Int +end + +@testitem "except de-duplicates the first source" begin + a = QueryOperators.query([1, 1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.except(a, empty)) == [1, 2] +end + +@testitem "except with everything excluded" begin + a = QueryOperators.query([1, 2]) + b = QueryOperators.query([1, 2, 3]) + + @test collect(QueryOperators.except(a, b)) == Int[] +end + +@testitem "except_by" begin + a = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=3, x="c")]) + b = QueryOperators.query([(a=2, x="ignored")]) + + res = QueryOperators.except_by(a, b, i -> i.a, :(i -> i.a)) + + @test collect(res) == [(a=1, x="a"), (a=3, x="c")] +end + +@testitem "intersect" begin + a = QueryOperators.query([1, 2, 2, 3, 4]) + b = QueryOperators.query([2, 4, 5]) + + res = QueryOperators.intersect(a, b) + + # Distinct elements of the first source that also occur in the second, + # in first-source order. + @test collect(res) == [2, 4] + @test eltype(res) == Int +end + +@testitem "intersect with no overlap" begin + a = QueryOperators.query([1, 2]) + b = QueryOperators.query([3, 4]) + + @test collect(QueryOperators.intersect(a, b)) == Int[] +end + +@testitem "intersect with an empty side" begin + a = QueryOperators.query([1, 2]) + empty = QueryOperators.query(Int[]) + + @test collect(QueryOperators.intersect(a, empty)) == Int[] + @test collect(QueryOperators.intersect(empty, a)) == Int[] +end + +@testitem "intersect_by" begin + a = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=2, x="bb")]) + b = QueryOperators.query([(a=2, x="ignored"), (a=9, x="ignored")]) + + res = QueryOperators.intersect_by(a, b, i -> i.a, :(i -> i.a)) + + # Distinct by key, so only the first a=2 element is kept. + @test collect(res) == [(a=2, x="b")] +end + +@testitem "set operators over DataValue elements treat nulls as equal" begin + using DataValues + + a = QueryOperators.query([DataValue{Int}(), DataValue(1), DataValue{Int}()]) + b = QueryOperators.query([DataValue(1)]) + + # Two nulls are a single distinct element, and 1 is excluded by the second + # sequence, so a single null survives. + res = collect(QueryOperators.except(a, b)) + @test length(res) == 1 + @test isna(res[1]) +end + +@testitem "set operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=1, v=2), (k=2, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + res = QueryOperators.union_by(grouped, grouped, g -> QueryOperators.key(g), :(g -> key(g))) + + # Unioning a grouping with itself by key is the identity on the groups. + collected = collect(res) + @test length(collected) == 2 + @test [QueryOperators.key(g) for g in collected] == [1, 2] +end diff --git a/test/test_enumerable_terminal.jl b/test/test_enumerable_terminal.jl new file mode 100644 index 0000000..b3bbd96 --- /dev/null +++ b/test/test_enumerable_terminal.jl @@ -0,0 +1,191 @@ +@testitem "any" begin + @test QueryOperators.any(QueryOperators.query([1, 2, 3])) == true + @test QueryOperators.any(QueryOperators.query(Int[])) == false + + source = QueryOperators.query([1, 2, 3]) + @test QueryOperators.any(source, i -> i > 2, :(i -> i > 2)) == true + @test QueryOperators.any(source, i -> i > 9, :(i -> i > 9)) == false + @test QueryOperators.any(QueryOperators.query(Int[]), i -> true, :(i -> true)) == false +end + +@testitem "all" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.all(source, i -> i > 0, :(i -> i > 0)) == true + @test QueryOperators.all(source, i -> i > 1, :(i -> i > 1)) == false + + # Vacuously true on an empty sequence, as in .NET. + @test QueryOperators.all(QueryOperators.query(Int[]), i -> false, :(i -> false)) == true +end + +@testitem "contains" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.contains(source, 2) == true + @test QueryOperators.contains(source, 9) == false + @test QueryOperators.contains(QueryOperators.query(Int[]), 1) == false +end + +@testitem "contains compares with isequal" begin + using DataValues + + @test QueryOperators.contains(QueryOperators.query([NaN, 1.0]), NaN) == true + + nulls = QueryOperators.query([DataValue{Int}(), DataValue(1)]) + @test QueryOperators.contains(nulls, DataValue{Int}()) == true +end + +@testitem "sequence_equal" begin + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2, 3]), QueryOperators.query([1, 2, 3])) == true + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2, 3]), QueryOperators.query([1, 2])) == false + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2]), QueryOperators.query([1, 2, 3])) == false + @test QueryOperators.sequence_equal(QueryOperators.query([1, 2]), QueryOperators.query([2, 1])) == false + @test QueryOperators.sequence_equal(QueryOperators.query(Int[]), QueryOperators.query(Int[])) == true +end + +@testitem "min_by and max_by" begin + source = QueryOperators.query([(a=2, x="b"), (a=1, x="a"), (a=3, x="c")]) + + # The whole element is returned, not the key. + @test QueryOperators.min_by(source, i -> i.a, :(i -> i.a)) == (a=1, x="a") + @test QueryOperators.max_by(source, i -> i.a, :(i -> i.a)) == (a=3, x="c") +end + +@testitem "min_by and max_by keep the first element on ties" begin + source = QueryOperators.query([(a=1, x="first"), (a=1, x="second")]) + + @test QueryOperators.min_by(source, i -> i.a, :(i -> i.a)) == (a=1, x="first") + @test QueryOperators.max_by(source, i -> i.a, :(i -> i.a)) == (a=1, x="first") +end + +@testitem "min_by and max_by reject an empty source" begin + source = QueryOperators.query(Int[]) + + @test_throws ErrorException QueryOperators.min_by(source, i -> i, :(i -> i)) + @test_throws ErrorException QueryOperators.max_by(source, i -> i, :(i -> i)) +end + +@testitem "aggregate without a seed" begin + source = QueryOperators.query([1, 2, 3, 4]) + + @test QueryOperators.aggregate(source, (a, b) -> a + b, :((a, b) -> a + b)) == 10 + @test QueryOperators.aggregate(QueryOperators.query([7]), (a, b) -> a + b, :((a, b) -> a + b)) == 7 +end + +@testitem "aggregate without a seed rejects an empty source" begin + source = QueryOperators.query(Int[]) + + @test_throws ErrorException QueryOperators.aggregate(source, (a, b) -> a + b, :((a, b) -> a + b)) +end + +@testitem "aggregate with a seed" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.aggregate(source, 100, (a, b) -> a + b, :((a, b) -> a + b)) == 106 + + # The seed alone is the result for an empty source. + @test QueryOperators.aggregate(QueryOperators.query(Int[]), 100, (a, b) -> a + b, :((a, b) -> a + b)) == 100 +end + +@testitem "aggregate with a seed of a different type" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.aggregate(source, "", (acc, cur) -> acc * string(cur), :((acc, cur) -> acc * string(cur))) + + @test res == "123" +end + +@testitem "first" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.first(source) == 1 + @test QueryOperators.first(source, i -> i > 1, :(i -> i > 1)) == 2 +end + +@testitem "first errors on an empty source or no match" begin + @test_throws ErrorException QueryOperators.first(QueryOperators.query(Int[])) + @test_throws ErrorException QueryOperators.first(QueryOperators.query([1, 2]), i -> i > 9, :(i -> i > 9)) +end + +@testitem "last" begin + source = QueryOperators.query([1, 2, 3]) + + @test QueryOperators.last(source) == 3 + @test QueryOperators.last(source, i -> i < 3, :(i -> i < 3)) == 2 +end + +@testitem "last errors on an empty source or no match" begin + @test_throws ErrorException QueryOperators.last(QueryOperators.query(Int[])) + @test_throws ErrorException QueryOperators.last(QueryOperators.query([1, 2]), i -> i > 9, :(i -> i > 9)) +end + +@testitem "last over NamedTuple rows" begin + source = QueryOperators.query([(a=1, x="a"), (a=2, x="b"), (a=3, x="c")]) + + @test QueryOperators.last(source) == (a=3, x="c") + @test QueryOperators.last(source, i -> i.a < 3, :(i -> i.a < 3)) == (a=2, x="b") +end + +@testitem "single" begin + @test QueryOperators.single(QueryOperators.query([7])) == 7 + @test QueryOperators.single(QueryOperators.query([1, 2, 3]), i -> i == 2, :(i -> i == 2)) == 2 +end + +@testitem "single errors unless exactly one element matches" begin + @test_throws ErrorException QueryOperators.single(QueryOperators.query(Int[])) + @test_throws ErrorException QueryOperators.single(QueryOperators.query([1, 2])) + + source = QueryOperators.query([1, 2, 3]) + @test_throws ErrorException QueryOperators.single(source, i -> i > 1, :(i -> i > 1)) + @test_throws ErrorException QueryOperators.single(source, i -> i > 9, :(i -> i > 9)) +end + +@testitem "element_at" begin + source = QueryOperators.query(["a", "b", "c"]) + + # 1-based, matching `index` and the rest of Julia. + @test QueryOperators.element_at(source, 1) == "a" + @test QueryOperators.element_at(source, 3) == "c" +end + +@testitem "element_at rejects out-of-range indices" begin + source = QueryOperators.query([1, 2, 3]) + + @test_throws ErrorException QueryOperators.element_at(source, 0) + @test_throws ErrorException QueryOperators.element_at(source, -1) + @test_throws ErrorException QueryOperators.element_at(source, 4) + @test_throws ErrorException QueryOperators.element_at(QueryOperators.query(Int[]), 1) +end + +@testitem "terminal operators compose with other operators" begin + source = QueryOperators.query([1, 2, 3, 4, 5, 6]) + + evens = QueryOperators.@filter(source, i -> i % 2 == 0) + + @test QueryOperators.first(evens) == 2 + @test QueryOperators.last(evens) == 6 + @test QueryOperators.aggregate(evens, (a, b) -> a + b, :((a, b) -> a + b)) == 12 + @test QueryOperators.any(evens) == true + @test QueryOperators.element_at(evens, 2) == 4 +end + +@testitem "terminal operators work downstream of groupby" begin + source = QueryOperators.query([(k=1, v=1), (k=2, v=2), (k=2, v=3)]) + grouped = QueryOperators.@groupby_simple(source, i -> i.k) + + biggest = QueryOperators.max_by(grouped, g -> length(g), :(g -> length(g))) + + @test QueryOperators.key(biggest) == 2 + @test length(biggest) == 2 + @test QueryOperators.count(grouped) == 2 +end + +@testitem "pivot_longer still works now that QueryOperators defines any" begin + # enumerable_pivot.jl calls Base.any on a generator; defining a query + # operator called `any` in the same module must not capture that call. + source = QueryOperators.query([(year=2017, US=1, EU=2), (year=2018, US=3, EU=4)]) + + res = QueryOperators.pivot_longer(source, (:US, :EU)) + + @test length(collect(res)) == 4 +end diff --git a/test/test_enumerable_typefiltering.jl b/test/test_enumerable_typefiltering.jl new file mode 100644 index 0000000..04a52dc --- /dev/null +++ b/test/test_enumerable_typefiltering.jl @@ -0,0 +1,89 @@ +@testitem "of_type" begin + source = QueryOperators.query(Any[1, "a", 2, "b", 3.0]) + + ints = QueryOperators.of_type(source, Int) + + @test collect(ints) == [1, 2] + @test eltype(ints) == Int + + strings = QueryOperators.of_type(source, String) + @test collect(strings) == ["a", "b"] + @test eltype(strings) == String +end + +@testitem "of_type keeps subtypes" begin + source = QueryOperators.query(Any[1, 2.0, "a"]) + + numbers = QueryOperators.of_type(source, Number) + + @test collect(numbers) == [1, 2.0] + @test eltype(numbers) == Number +end + +@testitem "of_type over a Union element type" begin + source = QueryOperators.query(Union{Int,String}[1, "a", 2]) + + @test collect(QueryOperators.of_type(source, Int)) == [1, 2] +end + +@testitem "of_type matching nothing or everything" begin + source = QueryOperators.query(Any[1, 2, 3]) + + @test collect(QueryOperators.of_type(source, String)) == String[] + @test collect(QueryOperators.of_type(source, Int)) == [1, 2, 3] +end + +@testitem "of_type on an empty source" begin + source = QueryOperators.query(Any[]) + + @test collect(QueryOperators.of_type(source, Int)) == Int[] +end + +@testitem "cast" begin + source = QueryOperators.query([1, 2, 3]) + + res = QueryOperators.cast(source, Float64) + + @test collect(res) == [1.0, 2.0, 3.0] + @test eltype(res) == Float64 + @test Base.IteratorSize(typeof(res)) == Base.HasLength() + @test length(res) == 3 +end + +@testitem "cast widens an Any source to a concrete type" begin + source = QueryOperators.query(Any[1, 2, 3]) + + res = QueryOperators.cast(source, Int) + + @test collect(res) == [1, 2, 3] + @test eltype(res) == Int +end + +@testitem "cast fails on an element that cannot be converted" begin + source = QueryOperators.query(Any[1, "a"]) + + res = QueryOperators.cast(source, Int) + + @test_throws MethodError collect(res) +end + +@testitem "cast fails on a lossy conversion" begin + source = QueryOperators.query([1.5]) + + @test_throws InexactError collect(QueryOperators.cast(source, Int)) +end + +@testitem "cast on an empty source" begin + source = QueryOperators.query(Int[]) + + @test collect(QueryOperators.cast(source, Float64)) == Float64[] +end + +@testitem "of_type and cast compose with other operators" begin + source = QueryOperators.query(Any[1, "a", 2, "b", 3]) + + res = QueryOperators.cast(QueryOperators.of_type(source, Int), Float64) + + @test collect(res) == [1.0, 2.0, 3.0] + @test eltype(res) == Float64 +end