Add the order, reverse, shuffle and index operators

Ports Enumerable.Order and OrderDescending (.NET 7), Enumerable.Shuffle
(.NET 10) and Enumerable.Index (.NET 9), plus Enumerable.Reverse.

order and order_descending are expressed in terms of orderby with an identity
key selector, so they produce an EnumerableOrderby and thenby can still follow
them.

index yields (index, item) NamedTuples with 1-based indices, matching Julia
rather than .NET's 0-based Index(). It is lazy; reverse and shuffle have to
collect their source first.

shuffle takes an optional RNG, which makes it reproducible; without one it
uses the default RNG and, as in .NET, is not cryptographically secure. This
adds Random to the dependencies.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: davidanthoff

Project.toml

@@ -7,6 +7,7 @@ DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 TableShowUtils = "5e66a065-1f0a-5976-b372-e0b8c017ca10"
 DataValues = "e7dc6d0d-1eca-5fa6-8ad6-5aecde8b7ea5"
 IteratorInterfaceExtensions = "82899510-4779-5014-852e-03e436cf321d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 
 [extras]
 TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
@@ -18,6 +19,7 @@ DataStructures = "0.11, 0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19"
 TableShowUtils = "0.1.1, 0.2, 0.3, 1"
 DataValues = "0.4.4, 0.5, 1"
 IteratorInterfaceExtensions = "0.1.1, 1"
+Random = "1"
 
 [targets]
 test = ["Test", "TestItemRunner"]

src/QueryOperators.jl

@@ -4,6 +4,7 @@ using DataStructures
 using IteratorInterfaceExtensions
 using TableShowUtils
 import DataValues
+import Random
 
 export Grouping, key
 
@@ -30,6 +31,9 @@ include("enumerable/enumerable_concat.jl")
 include("enumerable/enumerable_union.jl")
 include("enumerable/enumerable_except.jl")
 include("enumerable/enumerable_intersect.jl")
+include("enumerable/enumerable_reverse.jl")
+include("enumerable/enumerable_shuffle.jl")
+include("enumerable/enumerable_index.jl")
 include("enumerable/enumerable_pivot.jl")
 include("enumerable/enumerable_summarize.jl")
 include("enumerable/show.jl")

src/enumerable/enumerable_index.jl

@@ -0,0 +1,33 @@
+struct EnumerableIndex{T,TI,S} <: Enumerable
+    source::S
+end
+
+# Enumerable.Index (.NET 9) pairs each element with its position. .NET yields
+# (Index, Item) tuples; the Julia equivalent is an (index, item) NamedTuple.
+# Indices are 1-based, matching the rest of Julia rather than .NET.
+function index(source::Enumerable)
+    TI = eltype(source)
+    T = NamedTuple{(:index, :item),Tuple{Int,TI}}
+    return EnumerableIndex{T,TI,typeof(source)}(source)
+end
+
+Base.IteratorSize(::Type{EnumerableIndex{T,TI,S}}) where {T,TI,S} = haslength(S)
+
+Base.eltype(::Type{EnumerableIndex{T,TI,S}}) where {T,TI,S} = T
+
+Base.length(iter::EnumerableIndex) = length(iter.source)
+
+function Base.iterate(iter::EnumerableIndex{T,TI,S}) where {T,TI,S}
+    ret = iterate(iter.source)
+    ret === nothing && return nothing
+
+    return T((1, ret[1])), (i=1, state=ret[2])
+end
+
+function Base.iterate(iter::EnumerableIndex{T,TI,S}, state) where {T,TI,S}
+    ret = iterate(iter.source, state.state)
+    ret === nothing && return nothing
+
+    i = state.i + 1
+    return T((i, ret[1])), (i=i, state=ret[2])
+end

src/enumerable/enumerable_orderby.jl

@@ -28,6 +28,13 @@ function orderby_descending(source::Enumerable, f::Function, f_expr::Expr)
     return EnumerableOrderby{T,typeof(source),KS,TKS}(source, f, true)
 end
 
+# Enumerable.Order/OrderDescending (.NET 7): sort by the elements themselves.
+# Expressed in terms of orderby so that `thenby` can still follow them — it
+# walks back up the source chain looking for an EnumerableOrderby.
+order(source::Enumerable) = orderby(source, identity, :(i -> i))
+
+order_descending(source::Enumerable) = orderby_descending(source, identity, :(i -> i))
+
 function Base.iterate(iter::EnumerableOrderby{T,S,KS,TKS}) where {T,S,KS,TKS}
     rows = (Base.IteratorSize(typeof(iter)) isa Base.HasLength || Base.IteratorSize(typeof(iter)) isa Base.HasShape) ? length(iter) : 0
 

src/enumerable/enumerable_reverse.jl

@@ -0,0 +1,34 @@
+struct EnumerableReverse{T,S} <: Enumerable
+    source::S
+end
+
+function reverse(source::Enumerable)
+    T = eltype(source)
+    return EnumerableReverse{T,typeof(source)}(source)
+end
+
+Base.IteratorSize(::Type{EnumerableReverse{T,S}}) where {T,S} = haslength(S)
+
+Base.eltype(::Type{EnumerableReverse{T,S}}) where {T,S} = T
+
+Base.length(iter::EnumerableReverse) = length(iter.source)
+
+# Reversing needs the whole source, so the elements are collected on the first
+# call and then handed out back to front.
+function Base.iterate(iter::EnumerableReverse{T,S}) where {T,S}
+    elements = Base.collect(T, iter.source)
+
+    if length(elements)==0
+        return nothing
+    end
+
+    return elements[end], (elements, length(elements)-1)
+end
+
+function Base.iterate(iter::EnumerableReverse{T,S}, state) where {T,S}
+    if state[2]<1
+        return nothing
+    else
+        return state[1][state[2]], (state[1], state[2]-1)
+    end
+end

src/enumerable/enumerable_shuffle.jl

@@ -0,0 +1,41 @@
+struct EnumerableShuffle{T,S,R} <: Enumerable
+    source::S
+    rng::R
+end
+
+function shuffle(source::Enumerable)
+    return shuffle(source, Random.default_rng())
+end
+
+function shuffle(source::Enumerable, rng::Random.AbstractRNG)
+    T = eltype(source)
+    return EnumerableShuffle{T,typeof(source),typeof(rng)}(source, rng)
+end
+
+Base.IteratorSize(::Type{EnumerableShuffle{T,S,R}}) where {T,S,R} = haslength(S)
+
+Base.eltype(::Type{EnumerableShuffle{T,S,R}}) where {T,S,R} = T
+
+Base.length(iter::EnumerableShuffle) = length(iter.source)
+
+# Like Enumerable.Shuffle, the randomisation is not cryptographically secure.
+# Pass an explicit RNG to make a shuffle reproducible.
+function Base.iterate(iter::EnumerableShuffle{T,S,R}) where {T,S,R}
+    elements = Base.collect(T, iter.source)
+
+    if length(elements)==0
+        return nothing
+    end
+
+    Random.shuffle!(iter.rng, elements)
+
+    return elements[1], (elements, 2)
+end
+
+function Base.iterate(iter::EnumerableShuffle{T,S,R}, state) where {T,S,R}
+    if state[2]>length(state[1])
+        return nothing
+    else
+        return state[1][state[2]], (state[1], state[2]+1)
+    end
+end

src/operators.jl

@@ -156,3 +156,17 @@ function except_by end
 function intersect end
 
 function intersect_by end
+
+# Ordering and row position. `order`/`order_descending` are Enumerable.Order and
+# OrderDescending (.NET 7); `shuffle` is Enumerable.Shuffle (.NET 10); `index`
+# is Enumerable.Index (.NET 9).
+
+function order end
+
+function order_descending end
+
+function reverse end
+
+function shuffle end
+
+function index end

test/runtests.jl

@@ -3,6 +3,7 @@ using TestItemRunner
 include("test_core.jl")
 include("test_enumerable_outerjoins.jl")
 include("test_enumerable_setops.jl")
+include("test_enumerable_ordering.jl")
 include("test_enumerable_unique.jl")
 include("test_enumerable_summarize.jl")
 include("test_namedtupleutilities.jl")

test/test_enumerable_ordering.jl

@@ -0,0 +1,141 @@
+@testitem "order" begin
+    source = QueryOperators.query([3, 1, 2])
+
+    res = QueryOperators.order(source)
+
+    @test collect(res) == [1, 2, 3]
+    @test eltype(res) == Int
+    @test Base.IteratorSize(typeof(res)) == Base.HasLength()
+    @test length(res) == 3
+end
+
+@testitem "order_descending" begin
+    source = QueryOperators.query([3, 1, 2])
+
+    @test collect(QueryOperators.order_descending(source)) == [3, 2, 1]
+end
+
+@testitem "order on an empty source" begin
+    source = QueryOperators.query(Int[])
+
+    @test collect(QueryOperators.order(source)) == Int[]
+    @test collect(QueryOperators.order_descending(source)) == Int[]
+end
+
+@testitem "thenby can follow order" begin
+    source = QueryOperators.query([(a=1, b=2), (a=1, b=1), (a=0, b=9)])
+
+    # order sorts by the whole element; thenby then refines it, which only
+    # works because order reuses EnumerableOrderby.
+    res = QueryOperators.@thenby(QueryOperators.order(source), i -> i.b)
+
+    @test collect(res) == [(a=0, b=9), (a=1, b=1), (a=1, b=2)]
+end
+
+@testitem "reverse" begin
+    source = QueryOperators.query([1, 2, 3])
+
+    res = QueryOperators.reverse(source)
+
+    @test collect(res) == [3, 2, 1]
+    @test eltype(res) == Int
+    @test Base.IteratorSize(typeof(res)) == Base.HasLength()
+    @test length(res) == 3
+end
+
+@testitem "reverse on empty and single-element sources" begin
+    @test collect(QueryOperators.reverse(QueryOperators.query(Int[]))) == Int[]
+    @test collect(QueryOperators.reverse(QueryOperators.query([7]))) == [7]
+end
+
+@testitem "reverse of reverse is the identity" begin
+    source = QueryOperators.query([1, 2, 3, 4])
+
+    @test collect(QueryOperators.reverse(QueryOperators.reverse(source))) == [1, 2, 3, 4]
+end
+
+@testitem "shuffle is a permutation of its source" begin
+    using Random
+
+    source = QueryOperators.query(collect(1:50))
+
+    res = QueryOperators.shuffle(source)
+
+    @test sort(collect(res)) == collect(1:50)
+    @test eltype(res) == Int
+    @test length(res) == 50
+end
+
+@testitem "shuffle with an explicit rng is reproducible" begin
+    using Random
+
+    source = QueryOperators.query(collect(1:50))
+
+    a = collect(QueryOperators.shuffle(source, MersenneTwister(42)))
+    b = collect(QueryOperators.shuffle(source, MersenneTwister(42)))
+
+    @test a == b
+    @test sort(a) == collect(1:50)
+end
+
+@testitem "shuffle on an empty source" begin
+    @test collect(QueryOperators.shuffle(QueryOperators.query(Int[]))) == Int[]
+end
+
+@testitem "index" begin
+    source = QueryOperators.query(["a", "b", "c"])
+
+    res = QueryOperators.index(source)
+
+    # 1-based, matching Julia rather than .NET's 0-based Index().
+    @test collect(res) == [(index=1, item="a"), (index=2, item="b"), (index=3, item="c")]
+    @test eltype(res) == NamedTuple{(:index, :item),Tuple{Int,String}}
+    @test Base.IteratorSize(typeof(res)) == Base.HasLength()
+    @test length(res) == 3
+end
+
+@testitem "index on an empty source" begin
+    source = QueryOperators.query(String[])
+
+    @test collect(QueryOperators.index(source)) == NamedTuple{(:index, :item),Tuple{Int,String}}[]
+end
+
+@testitem "index is lazy" begin
+    import IteratorInterfaceExtensions
+
+    mutable struct CountedIndexSource
+        data::Vector{Int}
+        pulled::Int
+    end
+    Base.eltype(::Type{CountedIndexSource}) = Int
+    Base.IteratorSize(::Type{CountedIndexSource}) = Base.HasLength()
+    Base.length(c::CountedIndexSource) = length(c.data)
+    function Base.iterate(c::CountedIndexSource, i=1)
+        i > length(c.data) && return nothing
+        c.pulled += 1
+        return c.data[i], i + 1
+    end
+    IteratorInterfaceExtensions.isiterable(::CountedIndexSource) = true
+    IteratorInterfaceExtensions.getiterator(c::CountedIndexSource) = c
+
+    src = CountedIndexSource([1, 2, 3, 4, 5], 0)
+    res = QueryOperators.index(QueryOperators.query(src))
+
+    it = iterate(res)
+    it = iterate(res, it[2])
+
+    @test it[1] == (index=2, item=2)
+    @test src.pulled == 2
+end
+
+@testitem "ordering operators work downstream of groupby" begin
+    source = QueryOperators.query([(k=2, v=1), (k=1, v=2), (k=3, v=3)])
+    grouped = QueryOperators.@groupby_simple(source, i -> i.k)
+
+    reversed = collect(QueryOperators.reverse(grouped))
+    @test [QueryOperators.key(g) for g in reversed] == [3, 1, 2]
+
+    indexed = collect(QueryOperators.index(grouped))
+    @test [i.index for i in indexed] == [1, 2, 3]
+    @test [QueryOperators.key(i.item) for i in indexed] == [2, 1, 3]
+end