diff --git a/NEWS.md b/NEWS.md index c413344..f248848 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,13 @@ # Query.jl v1.2.0 Release Notes * Add @summarize for grouped and whole-table aggregation +* Add @left_join, @right_join and @full_join +* Add @concat, @union, @union_by, @except, @except_by, @intersect and @intersect_by +* Add @take_while, @drop_while, @take_last and @drop_last +* Add @order, @order_descending, @reverse, @shuffle and @index +* Add @append, @prepend and @zip +* Add @count_by, @aggregate_by and @chunk +* Add @of_type and @cast +* Add @min_by, @max_by, @any, @all, @contains, @sequence_equal, @aggregate, @first, @last, @single and @element_at # Query.jl v1.1.0 Release Notes * Add @pivot_wider and @pivot_longer diff --git a/Project.toml b/Project.toml index 02db8c9..e9d4704 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "Query" uuid = "1a8c2f83-1ff3-5112-b086-8aa67b057ba1" -version = "1.1.1-DEV" +version = "1.2.0-DEV" [deps] IterableTables = "1c8ee90f-4401-5389-894e-7a04a3dc0f4d" @@ -23,7 +23,7 @@ IndexedTables = "6deec6e2-d858-57c5-ab9b-e6ca5bd20e43" IterableTables = "0.8.2, 0.9, 0.10, 0.11, 1" Statistics = "1.11.5, 1.10" julia = "1.10" -QueryOperators = "1.1" +QueryOperators = "1.2" DataValues = "0.4.4, 0.5, 1" MacroTools = "0.4.4, 0.5" diff --git a/docs/src/standalonequerycommands.md b/docs/src/standalonequerycommands.md index 6589c5f..6e7eb71 100644 --- a/docs/src/standalonequerycommands.md +++ b/docs/src/standalonequerycommands.md @@ -657,3 +657,444 @@ result = long |> @pivot_wider(:country, :value) |> DataFrame # 1 │ 2017 1 2 # 2 │ 2018 3 4 ``` + +## The `@left_join`, `@right_join` and `@full_join` commands + +These commands have the form `source |> @left_join(inner, outerKeySelector, innerKeySelector, resultSelector)`, and correspond to `Enumerable.LeftJoin`, `RightJoin` and `FullJoin` in .NET. They take the same arguments as the `@join` command, but keep rows that have no match on the other side. + +`@left_join` keeps every row of `source`, `@right_join` keeps every row of `inner`, and `@full_join` keeps every row of both. Where there is no match, the missing side is filled with a `DataValue` that has no value — never with `missing`. `@full_join` emits the rows of `source` first, in source order, followed by the rows of `inner` whose key never appeared in `source`. + +#### Example + +```jldoctest +using Query + +people = [(id=1, name="John"), (id=2, name="Sally"), (id=3, name="Kirk")] +pets = [(owner=1, pet="Judy"), (owner=3, pet="Ruff")] + +q = people |> @left_join(pets, _.id, _.owner, {_.name, __.pet}) |> collect + +for row in q + println(row.name, ": ", row.pet) +end + +# output + +John: DataValue{String}("Judy") +Sally: DataValue{String}() +Kirk: DataValue{String}("Ruff") +``` + +## The `@concat`, `@union`, `@except` and `@intersect` commands + +These commands have the form `source |> @union(other)`, and correspond to `Enumerable.Concat`, `Union`, `Except` and `Intersect`. Both sequences must have the same element type. + +`@concat` appends `other` to `source`, keeping duplicates. The other three return distinct results, as they do in .NET: `@union` yields every element of either sequence, `@except` the elements of `source` that do not occur in `other`, and `@intersect` the elements that occur in both. Elements are compared with `isequal`, so two `DataValue`s that hold no value count as equal. + +#### Example + +```jldoctest +using Query + +a = [1,2,2,3] +b = [3,4] + +println(a |> @concat(b) |> collect) +println(a |> @union(b) |> collect) +println(a |> @except(b) |> collect) +println(a |> @intersect(b) |> collect) + +# output + +[1, 2, 2, 3, 3, 4] +[1, 2, 3, 4] +[1, 2] +[3] +``` + +## The `@union_by`, `@except_by` and `@intersect_by` commands + +These commands have the form `source |> @union_by(other, keySelector)`, and are the key-based versions of `@union`, `@except` and `@intersect`: two elements count as the same when their keys are equal. + +Note that this deviates from .NET. `Enumerable.ExceptBy` and `IntersectBy` take a sequence of *keys* as their second argument, while `UnionBy` takes a sequence of elements. Here all three take a sequence of elements and apply the key selector to both sequences, which keeps them consistent with each other and matches the equivalent SQL. + +`@union_by` keeps the first element seen for each key. + +#### Example + +```jldoctest +using Query + +a = [(k=1, v="a"), (k=2, v="b"), (k=3, v="c")] +b = [(k=2, v="B")] + +println(a |> @except_by(b, _.k) |> collect) +println(a |> @intersect_by(b, _.k) |> collect) + +# output + +[(k = 1, v = "a"), (k = 3, v = "c")] +[(k = 2, v = "b")] +``` + +## The `@take_while` and `@drop_while` commands + +These commands have the form `source |> @take_while(condition)`, and correspond to `Enumerable.TakeWhile` and `SkipWhile`. + +`@take_while` yields elements until `condition` first fails and then stops, so later elements are not returned even if they would satisfy it. `@drop_while` discards that same leading run and yields everything after it. + +#### Example + +```jldoctest +using Query + +source = [1,2,3,4,1,2] + +println(source |> @take_while(_ < 3) |> collect) +println(source |> @drop_while(_ < 3) |> collect) + +# output + +[1, 2] +[3, 4, 1, 2] +``` + +## The `@take_last` and `@drop_last` commands + +These commands have the form `source |> @take_last(n)`, and correspond to `Enumerable.TakeLast` and `SkipLast`. `@take_last` keeps the last `n` elements and `@drop_last` discards them. + +A count of zero or less yields nothing for `@take_last` and leaves the source unchanged for `@drop_last`, as in .NET. + +#### Example + +```jldoctest +using Query + +source = [1,2,3,4,5] + +println(source |> @take_last(2) |> collect) +println(source |> @drop_last(2) |> collect) + +# output + +[4, 5] +[1, 2, 3] +``` + +## The `@order` and `@order_descending` commands + +These commands have the form `source |> @order()`, and correspond to `Enumerable.Order` and `OrderDescending`. They sort by the elements themselves rather than by a key, so unlike `@orderby` they take no selector. `@thenby` and `@thenby_descending` can still follow them. + +#### Example + +```jldoctest +using Query + +source = [3,1,2] + +println(source |> @order() |> collect) +println(source |> @order_descending() |> collect) + +# output + +[1, 2, 3] +[3, 2, 1] +``` + +## The `@reverse` command + +The `@reverse` command has the form `source |> @reverse()`, and corresponds to `Enumerable.Reverse`. It yields the elements of the source in the opposite order. The whole source has to be read before the first element can be returned. + +#### Example + +```jldoctest +using Query + +source = [1,2,3] + +println(source |> @reverse() |> collect) + +# output + +[3, 2, 1] +``` + +## The `@shuffle` command + +The `@shuffle` command has the form `source |> @shuffle()`, and corresponds to `Enumerable.Shuffle`. It yields the elements of the source in a random order, using a random number generator that is not cryptographically secure. + +An explicit generator can be passed as the keyword argument `rng`, which makes a shuffle reproducible: `source |> @shuffle(rng=MersenneTwister(42))`. It is a keyword rather than a positional argument so that a single positional argument is unambiguously the source. + +#### Example + +```jldoctest +using Query +using Random + +source = [1,2,3,4,5] + +q = source |> @shuffle(rng=MersenneTwister(42)) |> collect + +println(sort(q)) + +# output + +[1, 2, 3, 4, 5] +``` + +## The `@index` command + +The `@index` command has the form `source |> @index()`, and corresponds to `Enumerable.Index`. It pairs each element with its position, yielding named tuples of the form `(index=..., item=...)`. Indices start at 1, matching the rest of Julia rather than .NET's zero-based `Index()`. + +#### Example + +```jldoctest +using Query + +source = ["a","b","c"] + +println(source |> @index() |> collect) + +# output + +[(index = 1, item = "a"), (index = 2, item = "b"), (index = 3, item = "c")] +``` + +## The `@append` and `@prepend` commands + +These commands have the form `source |> @append(element)`, and correspond to `Enumerable.Append` and `Prepend`. They add a single element after or before the elements of the source. The element is converted to the source's element type, so appending an `Int` to a sequence of `Float64` works. + +#### Example + +```jldoctest +using Query + +source = [2,3] + +println(source |> @append(4) |> collect) +println(source |> @prepend(1) |> collect) + +# output + +[2, 3, 4] +[1, 2, 3] +``` + +## The `@zip` command + +The `@zip` command has the form `source |> @zip(other)`, and corresponds to `Enumerable.Zip`. It pairs elements of the two sequences by position, yielding tuples, and stops at the shorter of the two — nothing is padded. + +A result selector can be given in the direct form, `@zip(source, other, resultSelector)`, where `_` refers to the element of `source` and `__` to the element of `other`. There is deliberately no piped form with a result selector, because it would be indistinguishable from the direct form without one; `source |> @zip(other) |> @map(...)` expresses the same thing. + +#### Example + +```jldoctest +using Query + +a = [1,2,3] +b = ["a","b"] + +println(a |> @zip(b) |> collect) + +# output + +[(1, "a"), (2, "b")] +``` + +## The `@count_by` command + +The `@count_by` command has the form `source |> @count_by(keySelector)`, and corresponds to `Enumerable.CountBy`. It counts how often each key occurs, without building the intermediate groups that `@groupby` would. + +The key columns are named as `@summarize` names them: a scalar key becomes a column called `key`, and a named tuple key contributes one column per field. The count is added as a column called `count`. + +#### Example + +```jldoctest +using Query + +source = [(k="a", v=1), (k="b", v=2), (k="a", v=3)] + +println(source |> @count_by(_.k) |> collect) + +# output + +[(key = "a", count = 2), (key = "b", count = 1)] +``` + +## The `@aggregate_by` command + +The `@aggregate_by` command has the form `source |> @aggregate_by(keySelector, seed, accumulator)`, and corresponds to `Enumerable.AggregateBy`. It folds the elements of each key into a single value, starting from `seed`. The accumulator is called as `accumulator(accumulated, element)`, matching .NET's argument order. + +Key columns are named as for `@count_by`, and the folded value is added as a column called `value`. For anything beyond a simple fold, `@summarize` is the more general and more idiomatic command. + +#### Example + +```jldoctest +using Query + +source = [(id="0", score=42), (id="1", score=5), (id="0", score=25)] + +println(source |> @aggregate_by(_.id, 0, (total, cur) -> total + cur.score) |> collect) + +# output + +[(key = "0", value = 67), (key = "1", value = 5)] +``` + +## The `@chunk` command + +The `@chunk` command has the form `source |> @chunk(n)`, and corresponds to `Enumerable.Chunk`. It splits the source into batches of at most `n` elements; the final batch is shorter when the source does not divide evenly. `n` must be at least 1. + +#### Example + +```jldoctest +using Query + +source = [1,2,3,4,5] + +println(source |> @chunk(2) |> collect) + +# output + +[[1, 2], [3, 4], [5]] +``` + +## The `@of_type` and `@cast` commands + +These commands have the form `source |> @of_type(T)`, and correspond to `Enumerable.OfType` and `Cast`. + +`@of_type` keeps only the elements that are instances of `T` and narrows the element type to `T`, which is useful when the source's element type is `Any` or a `Union`. `@cast` converts every element to `T`; .NET's `Cast` is a type assertion, but the Julia counterpart is a conversion, so it fails the way `convert` would on an element that cannot be represented as `T`. + +#### Example + +```jldoctest +using Query + +source = Any[1, "a", 2] + +println(source |> @of_type(Int) |> collect) +println(source |> @of_type(Int) |> @cast(Float64) |> collect) + +# output + +[1, 2] +[1.0, 2.0] +``` + +## The `@any`, `@all` and `@contains` commands + +These commands return a `Bool` rather than another query, and correspond to `Enumerable.Any`, `All` and `Contains`. + +`@any()` reports whether the source has any elements, and `@any(source, condition)` whether any element satisfies the condition. `@all(condition)` reports whether every element does, and is vacuously true for an empty source. `@contains(value)` reports whether the source contains `value`, comparing with `isequal`. + +As with `@count`, `@any` has no piped form taking a condition; write `source |> @filter(condition) |> @any()` instead. + +#### Example + +```jldoctest +using Query + +source = [1,2,3] + +println(source |> @any()) +println(@any(source, _ > 2)) +println(source |> @all(_ > 0)) +println(source |> @contains(2)) + +# output + +true +true +true +true +``` + +## The `@first`, `@last`, `@single` and `@element_at` commands + +These commands return a single element rather than another query, and correspond to `Enumerable.First`, `Last`, `Single` and `ElementAt`. + +`@first()` and `@last()` return the first and last element, and error if the source is empty. `@single()` returns the only element and errors unless there is exactly one. Each also has a direct form taking a condition, such as `@first(source, condition)`. `@element_at(n)` returns the element at position `n`, counting from 1 as the rest of Julia does rather than from 0 as .NET's `ElementAt` does. + +#### Example + +```jldoctest +using Query + +source = [1,2,3,4] + +println(source |> @first()) +println(source |> @last()) +println(source |> @element_at(2)) +println(@single(source, _ == 3)) + +# output + +1 +4 +2 +3 +``` + +## The `@min_by` and `@max_by` commands + +These commands have the form `source |> @min_by(keySelector)`, and correspond to `Enumerable.MinBy` and `MaxBy`. They return the *element* whose key is smallest or largest, not the key itself. Ties keep the first such element, as in .NET, and an empty source is an error. + +#### Example + +```jldoctest +using Query + +source = [(a=2, x="b"), (a=1, x="a"), (a=3, x="c")] + +println(source |> @min_by(_.a)) +println(source |> @max_by(_.a)) + +# output + +(a = 1, x = "a") +(a = 3, x = "c") +``` + +## The `@aggregate` command + +The `@aggregate` command has the form `source |> @aggregate(accumulator)`, and corresponds to `Enumerable.Aggregate`. It folds the source into a single value, calling `accumulator(accumulated, element)`. + +Without a seed the fold starts from the first element and an empty source is an error. A seed can be given as the keyword argument `seed`, as in `source |> @aggregate(accumulator, seed=0)`, in which case an empty source yields the seed. It is a keyword rather than a positional argument so that the piped and direct forms can be told apart. + +For aggregating a table, `@summarize` is the more idiomatic command. + +#### Example + +```jldoctest +using Query + +source = [1,2,3,4] + +println(source |> @aggregate((acc, cur) -> acc + cur)) +println(source |> @aggregate((acc, cur) -> acc + cur, seed=100)) + +# output + +10 +110 +``` + +## The `@sequence_equal` command + +The `@sequence_equal` command has the form `source |> @sequence_equal(other)`, and corresponds to `Enumerable.SequenceEqual`. It reports whether the two sequences have the same elements in the same order, comparing with `isequal`. + +#### Example + +```jldoctest +using Query + +source = [1,2,3] + +println(source |> @sequence_equal([1,2,3])) +println(source |> @sequence_equal([1,2])) + +# output + +true +false +``` diff --git a/src/Query.jl b/src/Query.jl index 718cdf7..3d6692f 100644 --- a/src/Query.jl +++ b/src/Query.jl @@ -11,6 +11,23 @@ export @map, @filter, @groupby, @orderby, @orderby_descending, @unique, @thenby, @thenby_descending, @groupjoin, @join, @mapmany, @take, @drop, @pivot_longer, @pivot_wider +export @left_join, @right_join, @full_join + +export @concat, @union, @union_by, @except, @except_by, @intersect, @intersect_by + +export @take_while, @drop_while, @take_last, @drop_last + +export @order, @order_descending, @reverse, @shuffle, @index + +export @append, @prepend, @zip + +export @count_by, @aggregate_by, @chunk + +export @of_type, @cast + +export @min_by, @max_by, @any, @all, @contains, @sequence_equal, @aggregate, + @first, @last, @single, @element_at + export @select, @rename, @mutate, @summarize, @disallowna, @dropna, @replacena # The following is a backwards compat fix diff --git a/src/standalone_query_macros.jl b/src/standalone_query_macros.jl index 2576d01..c8c25c3 100644 --- a/src/standalone_query_macros.jl +++ b/src/standalone_query_macros.jl @@ -409,3 +409,566 @@ end macro pivot_wider(names_from, values_from) return :(i -> QueryOperators.pivot_wider(QueryOperators.query(i), $(esc(names_from)), $(esc(values_from)))) end + +# Outer joins. As with @join, the direct form takes the outer sequence first and +# the piped form supplies it. + +macro left_join(outer, inner, outerKeySelector, innerKeySelector, resultSelector) + outerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(outerKeySelector) + innerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(innerKeySelector) + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + + q_outerKeySelector = Expr(:quote, outerKeySelector_as_anonym_func) + q_innerKeySelector = Expr(:quote, innerKeySelector_as_anonym_func) + q_resultSelector = Expr(:quote, resultSelector_as_anonym_func) + + return :(QueryOperators.left_join(QueryOperators.query($(esc(outer))), + QueryOperators.query($(esc(inner))), + $(esc(outerKeySelector_as_anonym_func)), $(esc(q_outerKeySelector)), + $(esc(innerKeySelector_as_anonym_func)), $(esc(q_innerKeySelector)), + $(esc(resultSelector_as_anonym_func)), $(esc(q_resultSelector)),)) |> + helper_namedtuples_replacement +end + +macro left_join(inner, outerKeySelector, innerKeySelector, resultSelector) + outerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(outerKeySelector) + innerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(innerKeySelector) + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + + q_outerKeySelector = Expr(:quote, outerKeySelector_as_anonym_func) + q_innerKeySelector = Expr(:quote, innerKeySelector_as_anonym_func) + q_resultSelector = Expr(:quote, resultSelector_as_anonym_func) + + return :( outer -> QueryOperators.left_join(QueryOperators.query(outer), + QueryOperators.query($(esc(inner))), + $(esc(outerKeySelector_as_anonym_func)), $(esc(q_outerKeySelector)), + $(esc(innerKeySelector_as_anonym_func)), $(esc(q_innerKeySelector)), + $(esc(resultSelector_as_anonym_func)), $(esc(q_resultSelector)),)) |> + helper_namedtuples_replacement +end + +macro right_join(outer, inner, outerKeySelector, innerKeySelector, resultSelector) + outerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(outerKeySelector) + innerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(innerKeySelector) + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + + q_outerKeySelector = Expr(:quote, outerKeySelector_as_anonym_func) + q_innerKeySelector = Expr(:quote, innerKeySelector_as_anonym_func) + q_resultSelector = Expr(:quote, resultSelector_as_anonym_func) + + return :(QueryOperators.right_join(QueryOperators.query($(esc(outer))), + QueryOperators.query($(esc(inner))), + $(esc(outerKeySelector_as_anonym_func)), $(esc(q_outerKeySelector)), + $(esc(innerKeySelector_as_anonym_func)), $(esc(q_innerKeySelector)), + $(esc(resultSelector_as_anonym_func)), $(esc(q_resultSelector)),)) |> + helper_namedtuples_replacement +end + +macro right_join(inner, outerKeySelector, innerKeySelector, resultSelector) + outerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(outerKeySelector) + innerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(innerKeySelector) + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + + q_outerKeySelector = Expr(:quote, outerKeySelector_as_anonym_func) + q_innerKeySelector = Expr(:quote, innerKeySelector_as_anonym_func) + q_resultSelector = Expr(:quote, resultSelector_as_anonym_func) + + return :( outer -> QueryOperators.right_join(QueryOperators.query(outer), + QueryOperators.query($(esc(inner))), + $(esc(outerKeySelector_as_anonym_func)), $(esc(q_outerKeySelector)), + $(esc(innerKeySelector_as_anonym_func)), $(esc(q_innerKeySelector)), + $(esc(resultSelector_as_anonym_func)), $(esc(q_resultSelector)),)) |> + helper_namedtuples_replacement +end + +macro full_join(outer, inner, outerKeySelector, innerKeySelector, resultSelector) + outerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(outerKeySelector) + innerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(innerKeySelector) + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + + q_outerKeySelector = Expr(:quote, outerKeySelector_as_anonym_func) + q_innerKeySelector = Expr(:quote, innerKeySelector_as_anonym_func) + q_resultSelector = Expr(:quote, resultSelector_as_anonym_func) + + return :(QueryOperators.full_join(QueryOperators.query($(esc(outer))), + QueryOperators.query($(esc(inner))), + $(esc(outerKeySelector_as_anonym_func)), $(esc(q_outerKeySelector)), + $(esc(innerKeySelector_as_anonym_func)), $(esc(q_innerKeySelector)), + $(esc(resultSelector_as_anonym_func)), $(esc(q_resultSelector)),)) |> + helper_namedtuples_replacement +end + +macro full_join(inner, outerKeySelector, innerKeySelector, resultSelector) + outerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(outerKeySelector) + innerKeySelector_as_anonym_func = helper_replace_anon_func_syntax(innerKeySelector) + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + + q_outerKeySelector = Expr(:quote, outerKeySelector_as_anonym_func) + q_innerKeySelector = Expr(:quote, innerKeySelector_as_anonym_func) + q_resultSelector = Expr(:quote, resultSelector_as_anonym_func) + + return :( outer -> QueryOperators.full_join(QueryOperators.query(outer), + QueryOperators.query($(esc(inner))), + $(esc(outerKeySelector_as_anonym_func)), $(esc(q_outerKeySelector)), + $(esc(innerKeySelector_as_anonym_func)), $(esc(q_innerKeySelector)), + $(esc(resultSelector_as_anonym_func)), $(esc(q_resultSelector)),)) |> + helper_namedtuples_replacement +end + +# Set operations over two sequences. + +macro concat(source, other) + return :(QueryOperators.concat(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))))) +end + +macro concat(other) + return :( i -> QueryOperators.concat(QueryOperators.query(i), QueryOperators.query($(esc(other))))) +end + +macro union(source, other) + return :(QueryOperators.union(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))))) +end + +macro union(other) + return :( i -> QueryOperators.union(QueryOperators.query(i), QueryOperators.query($(esc(other))))) +end + +macro except(source, other) + return :(QueryOperators.except(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))))) +end + +macro except(other) + return :( i -> QueryOperators.except(QueryOperators.query(i), QueryOperators.query($(esc(other))))) +end + +macro intersect(source, other) + return :(QueryOperators.intersect(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))))) +end + +macro intersect(other) + return :( i -> QueryOperators.intersect(QueryOperators.query(i), QueryOperators.query($(esc(other))))) +end + +# Set operations that compare a key rather than whole elements. Unlike +# Enumerable.ExceptBy and IntersectBy, whose second argument is a bare sequence +# of keys, the selector here applies to both sequences. + +macro union_by(source, other, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.union_by(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))), + $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro union_by(other, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.union_by(QueryOperators.query(i), QueryOperators.query($(esc(other))), + $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro except_by(source, other, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.except_by(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))), + $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro except_by(other, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.except_by(QueryOperators.query(i), QueryOperators.query($(esc(other))), + $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro intersect_by(source, other, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.intersect_by(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))), + $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro intersect_by(other, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.intersect_by(QueryOperators.query(i), QueryOperators.query($(esc(other))), + $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +# Partitioning. + +macro take_while(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.take_while(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro take_while(f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.take_while(QueryOperators.query(i), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro drop_while(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.drop_while(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro drop_while(f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.drop_while(QueryOperators.query(i), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro take_last(source, n) + return :(QueryOperators.take_last(QueryOperators.query($(esc(source))), $(esc(n)))) +end + +macro take_last(n) + return :( i -> QueryOperators.take_last(QueryOperators.query(i), $(esc(n)))) +end + +macro drop_last(source, n) + return :(QueryOperators.drop_last(QueryOperators.query($(esc(source))), $(esc(n)))) +end + +macro drop_last(n) + return :( i -> QueryOperators.drop_last(QueryOperators.query(i), $(esc(n)))) +end + +# Ordering and row position. @order sorts by whole elements, so it takes no +# selector; @thenby can still follow it. + +macro order(source) + return :(QueryOperators.order(QueryOperators.query($(esc(source))))) +end + +macro order() + return :( i -> QueryOperators.order(QueryOperators.query(i))) +end + +macro order_descending(source) + return :(QueryOperators.order_descending(QueryOperators.query($(esc(source))))) +end + +macro order_descending() + return :( i -> QueryOperators.order_descending(QueryOperators.query(i))) +end + +macro reverse(source) + return :(QueryOperators.reverse(QueryOperators.query($(esc(source))))) +end + +macro reverse() + return :( i -> QueryOperators.reverse(QueryOperators.query(i))) +end + +macro index(source) + return :(QueryOperators.index(QueryOperators.query($(esc(source))))) +end + +macro index() + return :( i -> QueryOperators.index(QueryOperators.query(i))) +end + +# Combining sequences. @append and @prepend add a single element. + +macro append(source, element) + return :(QueryOperators.append(QueryOperators.query($(esc(source))), $(esc(element)))) +end + +macro append(element) + return :( i -> QueryOperators.append(QueryOperators.query(i), $(esc(element)))) +end + +macro prepend(source, element) + return :(QueryOperators.prepend(QueryOperators.query($(esc(source))), $(esc(element)))) +end + +macro prepend(element) + return :( i -> QueryOperators.prepend(QueryOperators.query(i), $(esc(element)))) +end + +# Keyed aggregation and batching. + +macro count_by(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.count_by(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro count_by(f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.count_by(QueryOperators.query(i), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro chunk(source, n) + return :(QueryOperators.chunk(QueryOperators.query($(esc(source))), $(esc(n)))) +end + +macro chunk(n) + return :( i -> QueryOperators.chunk(QueryOperators.query(i), $(esc(n)))) +end + +# Type filtering. + +macro of_type(source, T) + return :(QueryOperators.of_type(QueryOperators.query($(esc(source))), $(esc(T)))) +end + +macro of_type(T) + return :( i -> QueryOperators.of_type(QueryOperators.query(i), $(esc(T)))) +end + +macro cast(source, T) + return :(QueryOperators.cast(QueryOperators.query($(esc(source))), $(esc(T)))) +end + +macro cast(T) + return :( i -> QueryOperators.cast(QueryOperators.query(i), $(esc(T)))) +end + +# Terminal operators, which return a value rather than another query. + +macro all(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.all(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro all(f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.all(QueryOperators.query(i), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro min_by(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.min_by(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro min_by(f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.min_by(QueryOperators.query(i), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro max_by(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.max_by(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro max_by(f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :( i -> QueryOperators.max_by(QueryOperators.query(i), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro contains(source, value) + return :(QueryOperators.contains(QueryOperators.query($(esc(source))), $(esc(value)))) +end + +macro contains(value) + return :( i -> QueryOperators.contains(QueryOperators.query(i), $(esc(value)))) +end + +macro element_at(source, n) + return :(QueryOperators.element_at(QueryOperators.query($(esc(source))), $(esc(n)))) +end + +macro element_at(n) + return :( i -> QueryOperators.element_at(QueryOperators.query(i), $(esc(n)))) +end + +macro sequence_equal(source, other) + return :(QueryOperators.sequence_equal(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))))) +end + +macro sequence_equal(other) + return :( i -> QueryOperators.sequence_equal(QueryOperators.query(i), QueryOperators.query($(esc(other))))) +end + +macro any(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.any(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro any(source) + return :(QueryOperators.any(QueryOperators.query($(esc(source))))) +end + +macro any() + return :( i -> QueryOperators.any(QueryOperators.query(i))) +end + +macro first(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.first(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro first(source) + return :(QueryOperators.first(QueryOperators.query($(esc(source))))) +end + +macro first() + return :( i -> QueryOperators.first(QueryOperators.query(i))) +end + +macro last(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.last(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro last(source) + return :(QueryOperators.last(QueryOperators.query($(esc(source))))) +end + +macro last() + return :( i -> QueryOperators.last(QueryOperators.query(i))) +end + +macro single(source, f) + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + return :(QueryOperators.single(QueryOperators.query($(esc(source))), $(esc(f_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement +end + +macro single(source) + return :(QueryOperators.single(QueryOperators.query($(esc(source))))) +end + +macro single() + return :( i -> QueryOperators.single(QueryOperators.query(i))) +end + +# Returns true when a macro argument is the keyword argument `name = value`. +function _is_named_kwarg(arg, name::Symbol) + return arg isa Expr && (arg.head == :(=) || arg.head == :kw) && + length(arg.args) == 2 && arg.args[1] == name +end + +# Splits macro arguments into positional ones and the value of a single +# optional keyword argument. The keyword form is what makes the piped and +# direct calls of @shuffle and @aggregate tell each other apart: a lone +# positional argument is always the source, never an rng or a seed. +function _split_kwarg(macro_name, args, name::Symbol) + kwargs = filter(a -> _is_named_kwarg(a, name), args) + positional = filter(a -> !_is_named_kwarg(a, name), args) + + length(kwargs) <= 1 || error("$macro_name accepts at most one `$name` argument") + for a in positional + if a isa Expr && (a.head == :(=) || a.head == :kw) + error("$macro_name does not accept the keyword argument `$(a.args[1])`") + end + end + + return positional, isempty(kwargs) ? nothing : kwargs[1].args[2] +end + +macro shuffle(args...) + positional, rng = _split_kwarg("@shuffle", args, :rng) + length(positional) <= 1 || error("@shuffle accepts at most one positional argument, the source") + + make_call(src) = rng === nothing ? + :(QueryOperators.shuffle(QueryOperators.query($src))) : + :(QueryOperators.shuffle(QueryOperators.query($src), $(esc(rng)))) + + if isempty(positional) + return :( i -> $(make_call(:i)) ) + else + return make_call(:($(esc(positional[1])))) + end +end + +macro aggregate(args...) + positional, seed = _split_kwarg("@aggregate", args, :seed) + 1 <= length(positional) <= 2 || + error("@aggregate takes the accumulator, optionally preceded by the source, plus an optional `seed=` argument") + + f = positional[end] + f_as_anonym_func = helper_replace_anon_func_syntax(f) + q = Expr(:quote, f_as_anonym_func) + + make_call(src) = seed === nothing ? + :(QueryOperators.aggregate(QueryOperators.query($src), $(esc(f_as_anonym_func)), $(esc(q)))) : + :(QueryOperators.aggregate(QueryOperators.query($src), $(esc(seed)), $(esc(f_as_anonym_func)), $(esc(q)))) + + result = length(positional) == 1 ? + :( i -> $(make_call(:i)) ) : + make_call(:($(esc(positional[1])))) + + return result |> helper_namedtuples_replacement +end + +macro aggregate_by(args...) + 3 <= length(args) <= 4 || + error("@aggregate_by takes a key selector, a seed and an accumulator, optionally preceded by the source") + + f, seed, accumulator = args[end-2], args[end-1], args[end] + + f_as_anonym_func = helper_replace_anon_func_syntax(f) + accumulator_as_anonym_func = helper_replace_anon_func_syntax(accumulator) + q = Expr(:quote, f_as_anonym_func) + + make_call(src) = :(QueryOperators.aggregate_by(QueryOperators.query($src), + $(esc(f_as_anonym_func)), $(esc(q)), $(esc(seed)), $(esc(accumulator_as_anonym_func)))) + + result = length(args) == 3 ? + :( i -> $(make_call(:i)) ) : + make_call(:($(esc(args[1])))) + + return result |> helper_namedtuples_replacement +end + +# @zip(other) / @zip(source, other) / @zip(source, other, resultSelector). +# There is deliberately no two-argument piped form with a result selector: it +# would be indistinguishable from the direct form, and `|> @zip(other) |> @map(...)` +# expresses the same thing. +macro zip(args...) + 1 <= length(args) <= 3 || + error("@zip takes the second sequence, optionally preceded by the source and optionally followed by a result selector") + + if length(args) == 1 + other = args[1] + return :( i -> QueryOperators.zip(QueryOperators.query(i), QueryOperators.query($(esc(other))))) + elseif length(args) == 2 + source, other = args + return :(QueryOperators.zip(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))))) + else + source, other, resultSelector = args + resultSelector_as_anonym_func = helper_replace_anon_func_syntax(resultSelector) + q = Expr(:quote, resultSelector_as_anonym_func) + return :(QueryOperators.zip(QueryOperators.query($(esc(source))), QueryOperators.query($(esc(other))), + $(esc(resultSelector_as_anonym_func)), $(esc(q)))) |> + helper_namedtuples_replacement + end +end diff --git a/test/runtests.jl b/test/runtests.jl index 8ed09fc..a570b7c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,6 +8,7 @@ include("test_pipesyntax.jl") include("test_macros.jl") include("test_standalone.jl") include("test_summarize.jl") +include("test_new_operators.jl") @run_package_tests diff --git a/test/test_new_operators.jl b/test/test_new_operators.jl new file mode 100644 index 0000000..3125756 --- /dev/null +++ b/test/test_new_operators.jl @@ -0,0 +1,268 @@ +@testitem "@left_join, @right_join and @full_join" begin + using DataFrames + using DataValues + + people = DataFrame(id=[1, 2, 3], name=["a", "b", "c"]) + pets = DataFrame(owner=[1, 3], pet=["cat", "dog"]) + + left = people |> @left_join(pets, _.id, _.owner, {_.name, __.pet}) |> DataFrame + @test size(left) == (3, 2) + @test left[!, :name] == ["a", "b", "c"] + @test isequal(left[!, :pet], ["cat", missing, "dog"]) + + right = people |> @right_join(pets, _.id, _.owner, {_.name, __.pet}) |> DataFrame + @test size(right) == (2, 2) + @test right[!, :pet] == ["cat", "dog"] + + full = people |> @full_join(pets, _.id, _.owner, {_.name, __.pet}) |> DataFrame + @test size(full) == (3, 2) + + # Direct form. + direct = DataFrame(@left_join(people, pets, _.id, _.owner, {_.name, __.pet})) + @test size(direct) == (3, 2) +end + +@testitem "outer joins produce DataValue, not missing, before the sink" begin + using DataFrames + using DataValues + + people = DataFrame(id=[1, 2], name=["a", "b"]) + pets = DataFrame(owner=[1], pet=["cat"]) + + rows = people |> @left_join(pets, _.id, _.owner, {_.name, __.pet}) |> collect + + @test length(rows) == 2 + @test rows[1].pet isa DataValue + @test isna(rows[2].pet) + @test !any(r -> ismissing(r.pet), rows) +end + +@testitem "@concat, @union, @except and @intersect" begin + using DataFrames + + a = DataFrame(x=[1, 2, 3]) + b = DataFrame(x=[3, 4]) + + @test (a |> @concat(b) |> DataFrame)[!, :x] == [1, 2, 3, 3, 4] + @test (a |> @union(b) |> DataFrame)[!, :x] == [1, 2, 3, 4] + @test (a |> @except(b) |> DataFrame)[!, :x] == [1, 2] + @test (a |> @intersect(b) |> DataFrame)[!, :x] == [3] + + # Direct form. + @test DataFrame(@union(a, b))[!, :x] == [1, 2, 3, 4] +end + +@testitem "@union_by, @except_by and @intersect_by" begin + using DataFrames + + a = DataFrame(k=[1, 2, 3], v=["a", "b", "c"]) + b = DataFrame(k=[2, 3], v=["B", "C"]) + + u = a |> @union_by(b, _.k) |> DataFrame + @test u[!, :k] == [1, 2, 3] + @test u[!, :v] == ["a", "b", "c"] + + @test (a |> @except_by(b, _.k) |> DataFrame)[!, :k] == [1] + @test (a |> @intersect_by(b, _.k) |> DataFrame)[!, :k] == [2, 3] + + @test DataFrame(@except_by(a, b, _.k))[!, :k] == [1] +end + +@testitem "@take_while and @drop_while" begin + using DataFrames + + df = DataFrame(x=[1, 2, 3, 4, 1]) + + @test (df |> @take_while(_.x < 3) |> DataFrame)[!, :x] == [1, 2] + @test (df |> @drop_while(_.x < 3) |> DataFrame)[!, :x] == [3, 4, 1] + + @test DataFrame(@take_while(df, _.x < 3))[!, :x] == [1, 2] +end + +@testitem "@take_last and @drop_last" begin + using DataFrames + + df = DataFrame(x=[1, 2, 3, 4, 5]) + + @test (df |> @take_last(2) |> DataFrame)[!, :x] == [4, 5] + @test (df |> @drop_last(2) |> DataFrame)[!, :x] == [1, 2, 3] + + @test DataFrame(@take_last(df, 2))[!, :x] == [4, 5] +end + +@testitem "@order and @order_descending" begin + using DataFrames + + df = DataFrame(x=[3, 1, 2]) + + @test (df |> @order() |> DataFrame)[!, :x] == [1, 2, 3] + @test (df |> @order_descending() |> DataFrame)[!, :x] == [3, 2, 1] + + @test DataFrame(@order(df))[!, :x] == [1, 2, 3] +end + +@testitem "@thenby can follow @order" begin + using DataFrames + + df = DataFrame(a=[1, 1, 0], b=[2, 1, 9]) + + res = df |> @order() |> @thenby(_.b) |> DataFrame + + @test res[!, :a] == [0, 1, 1] + @test res[!, :b] == [9, 1, 2] +end + +@testitem "@reverse, @shuffle and @index" begin + using DataFrames + using Random + + df = DataFrame(x=[1, 2, 3]) + + @test (df |> @reverse() |> DataFrame)[!, :x] == [3, 2, 1] + @test DataFrame(@reverse(df))[!, :x] == [3, 2, 1] + + @test sort((df |> @shuffle() |> DataFrame)[!, :x]) == [1, 2, 3] + + # The rng is a keyword, so a lone positional argument is still the source. + a = df |> @shuffle(rng=MersenneTwister(42)) |> DataFrame + b = df |> @shuffle(rng=MersenneTwister(42)) |> DataFrame + @test a[!, :x] == b[!, :x] + @test sort(a[!, :x]) == [1, 2, 3] + @test sort(DataFrame(@shuffle(df))[!, :x]) == [1, 2, 3] + + indexed = df |> @index() |> collect + @test [i.index for i in indexed] == [1, 2, 3] + @test [i.item.x for i in indexed] == [1, 2, 3] +end + +@testitem "@append and @prepend" begin + using DataFrames + + df = DataFrame(x=[2, 3]) + + @test (df |> @append((x=4,)) |> DataFrame)[!, :x] == [2, 3, 4] + @test (df |> @prepend((x=1,)) |> DataFrame)[!, :x] == [1, 2, 3] + @test DataFrame(@append(df, (x=4,)))[!, :x] == [2, 3, 4] +end + +@testitem "@zip" begin + using DataFrames + + a = DataFrame(x=[1, 2, 3]) + b = DataFrame(y=["a", "b"]) + + # Truncates to the shorter source. + zipped = a |> @zip(b) |> collect + @test length(zipped) == 2 + @test zipped[1] == ((x=1,), (y="a",)) + + with_selector = @zip(a, b, {v = _.x, w = __.y}) |> DataFrame + @test size(with_selector) == (2, 2) + @test with_selector[!, :v] == [1, 2] + @test with_selector[!, :w] == ["a", "b"] +end + +@testitem "@count_by, @aggregate_by and @chunk" begin + using DataFrames + + df = DataFrame(k=["a", "b", "a"], v=[1, 2, 3]) + + counted = df |> @count_by(_.k) |> DataFrame + @test counted[!, :key] == ["a", "b"] + @test counted[!, :count] == [2, 1] + + aggregated = df |> @aggregate_by(_.k, 0, (acc, cur) -> acc + cur.v) |> DataFrame + @test aggregated[!, :key] == ["a", "b"] + @test aggregated[!, :value] == [4, 2] + + chunks = df |> @chunk(2) |> collect + @test length(chunks) == 2 + @test length(chunks[1]) == 2 + @test length(chunks[2]) == 1 + + @test DataFrame(@count_by(df, _.k))[!, :count] == [2, 1] +end + +@testitem "@count_by agrees with @groupby plus a count" begin + using DataFrames + + df = DataFrame(k=[1, 2, 1, 3, 1]) + + by_count = df |> @count_by(_.k) |> DataFrame + by_group = df |> @groupby(_.k) |> @map({key = key(_), count = length(_)}) |> DataFrame + + @test by_count == by_group +end + +@testitem "@of_type and @cast" begin + source = Any[1, "a", 2] + + @test (source |> @of_type(Int) |> collect) == [1, 2] + @test (source |> @of_type(Int) |> @cast(Float64) |> collect) == [1.0, 2.0] + @test (@of_type(source, String) |> collect) == ["a"] +end + +@testitem "terminal operators" begin + using DataFrames + + df = DataFrame(x=[1, 2, 3, 4]) + + @test (df |> @any()) == true + @test @any(df, _.x > 3) == true + @test @any(df, _.x > 9) == false + @test (df |> @all(_.x > 0)) == true + @test (df |> @all(_.x > 1)) == false + + @test (df |> @first()).x == 1 + @test (df |> @last()).x == 4 + @test (df |> @element_at(2)).x == 2 + @test (df |> @min_by(_.x)).x == 1 + @test (df |> @max_by(_.x)).x == 4 + @test @single(df, _.x == 3).x == 3 + + @test (df |> @contains((x=2,))) == true + @test (df |> @contains((x=9,))) == false + @test (df |> @sequence_equal(df)) == true + @test (df |> @sequence_equal(DataFrame(x=[1, 2]))) == false +end + +@testitem "@aggregate with and without a seed" begin + using DataFrames + + df = DataFrame(x=[1, 2, 3]) + + # Without a seed the fold starts from the first element, so it folds rows. + @test (df |> @aggregate((acc, cur) -> (x = acc.x + cur.x,))).x == 6 + + # The seed is a keyword, which is what keeps the piped and direct forms apart. + @test (df |> @aggregate((acc, cur) -> acc + cur.x, seed=100)) == 106 + @test @aggregate(df, (acc, cur) -> acc + cur.x, seed=0) == 6 + @test @aggregate(df, (acc, cur) -> (x = acc.x + cur.x,)).x == 6 +end + +@testitem "new operators compose in a pipeline" begin + using DataFrames + + df = DataFrame(k=["a", "b", "a", "c"], v=[1, 2, 3, 4]) + + res = df |> + @filter(_.v < 4) |> + @count_by(_.k) |> + @order_descending() |> + @take_last(1) |> + DataFrame + + @test size(res) == (1, 2) +end + +@testitem "new operators work downstream of @groupby" begin + using DataFrames + + df = DataFrame(k=[1, 1, 2], v=[1, 2, 3]) + + groups = df |> @groupby(_.k) |> @reverse() |> collect + @test [key(g) for g in groups] == [2, 1] + + biggest = df |> @groupby(_.k) |> @max_by(length(_)) + @test key(biggest) == 1 +end