From 5c06bd2a63a3b1f489b950a1f28b6d5682f7f85b Mon Sep 17 00:00:00 2001 From: Jeff Chung Date: Wed, 2 Sep 2026 17:22:05 +0800 Subject: [PATCH 1/5] feat: add self-host splitmix64 random generator --- datasketches/src/common/mod.rs | 2 + datasketches/src/common/random.rs | 142 ++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 datasketches/src/common/random.rs diff --git a/datasketches/src/common/mod.rs b/datasketches/src/common/mod.rs index 918c57b5..c9c1cf79 100644 --- a/datasketches/src/common/mod.rs +++ b/datasketches/src/common/mod.rs @@ -26,3 +26,5 @@ pub use self::search_criteria::SearchCriteria; #[cfg(any(feature = "cpc", feature = "hll"))] pub(crate) mod inv_pow2; +#[cfg(any(feature = "kll", feature = "req"))] +pub(crate) mod random; diff --git a/datasketches/src/common/random.rs b/datasketches/src/common/random.rs new file mode 100644 index 00000000..e8443e35 --- /dev/null +++ b/datasketches/src/common/random.rs @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A thread-local source of uniform random bits. +//! +//! The generator is SplitMix64, the fixed-increment form of the splittable +//! generator introduced in and shipped +//! in the JDK as `java.util.SplittableRandom`. The constants and shift amounts +//! used here match Sebastiano Vigna's public-domain reference +//! implementation, . + +use std::cell::Cell; +use std::collections::hash_map::RandomState; +use std::hash::BuildHasher; + +/// Advances a SplitMix64 state by one step, returning `(next_state, output)`. +/// +/// The state advances by the fixed odd increment `0x9E37_79B9_7F4A_7C15`, the +/// 64-bit approximation of 2^64/φ, which makes the state a Weyl sequence of +/// full period 2^64. The output is that new state run through Stafford's +/// variant-13 mixer, a bijection, so the outputs inherit both the period and +/// the equidistribution of the state. +/// +/// The step is bit-for-bit identical to `next()` in the reference +/// `splitmix64.c`, so a given seed yields the same stream as that +/// implementation. +fn next_u64(state: u64) -> (u64, u64) { + let state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + (state, z ^ (z >> 31)) +} + +/// Returns the seed for the calling thread's stream. +/// +/// The seed comes from the operating-system entropy that `std` already holds to +/// key its hash maps, mixed with the current thread's id so that threads in one +/// process start at unrelated points of the cycle. +fn random_seed() -> u64 { + RandomState::new().hash_one(std::thread::current().id()) +} + +thread_local! { + /// The calling thread's SplitMix64 state, seeded on first use. + static STATE: Cell = Cell::new(random_seed()); +} + +/// Returns `true` or `false`, each with probability 1/2. +/// +/// The bit is the low bit of the next output of the calling thread's stream. +/// The call is a handful of arithmetic operations on thread-local state: it +/// never allocates, locks, or blocks. The first call on a thread seeds that +/// thread's state. +pub(crate) fn random_bit() -> bool { + STATE.with(|state| { + let (next, value) = next_u64(state.get()); + state.set(next); + value & 1 == 1 + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn assert_split_mix64_matches_the_reference_output() { + // Expected outputs for seed 0, taken from the reference splitmix64.c. + let expected = [ + 0xe220_a839_7b1d_cdaf, + 0x6e78_9e6a_a1b9_65f4, + 0x06c4_5d18_8009_454f, + 0xf88b_b8a8_724c_81ec, + ]; + let mut state = 0; + for want in expected { + let (next, got) = next_u64(state); + state = next; + assert_eq!(got, want); + } + } + + /// A coin that stops moving is the failure this module is most exposed to: + /// dropping the write-back in `random_bit` pins every flip to one value, and + /// the sketches keep compacting and keep passing their own tests while + /// always promoting the same half. + #[test] + fn assert_random_bit_yields_both_values() { + let mut seen = [false; 2]; + for _ in 0..64 { + seen[usize::from(random_bit())] = true; + } + assert_eq!(seen, [true, true], "the stream stopped advancing"); + } + + /// The KLL and REQ error bounds hold only for a fair coin, so pin the + /// balance. One million draws have a standard deviation of 500 heads; the + /// bound below is six of them, which a correct generator exceeds about twice + /// in a billion runs. + #[test] + fn assert_random_bit_is_unbiased() { + const DRAWS: u32 = 1_000_000; + const TOLERANCE: u32 = 3_000; + + let mut heads = 0; + for _ in 0..DRAWS { + heads += u32::from(random_bit()); + } + assert!(heads.abs_diff(DRAWS / 2) <= TOLERANCE, "heads = {heads}"); + } + + /// Threads must not share a starting point, otherwise sketches filled on + /// different threads make identical compaction choices and their errors + /// correlate. Two independent streams agree on 64 consecutive bits with + /// probability 2^-64. + #[test] + fn assert_each_thread_draws_its_own_stream() { + fn draw_word() -> u64 { + (0..64).fold(0, |word, _| (word << 1) | u64::from(random_bit())) + } + + let other = std::thread::spawn(draw_word) + .join() + .expect("the drawing thread should not panic"); + assert_ne!(draw_word(), other, "two threads shared one stream"); + } +} From 3d68dbb308d3fce4fb28ef105b77d9e0af9d0a90 Mon Sep 17 00:00:00 2001 From: Jeff Chung Date: Wed, 2 Sep 2026 17:22:41 +0800 Subject: [PATCH 2/5] chore: remove rand crate dep and switch to our self written random_bit instead --- Cargo.lock | 1 - datasketches/Cargo.toml | 7 ++----- datasketches/src/kll/sketch.rs | 5 +++-- datasketches/src/req/compactor.rs | 3 ++- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 106a6632..ab02dc69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -244,7 +244,6 @@ version = "0.5.0" dependencies = [ "googletest", "insta", - "rand", ] [[package]] diff --git a/datasketches/Cargo.toml b/datasketches/Cargo.toml index 1033540e..1a31d981 100644 --- a/datasketches/Cargo.toml +++ b/datasketches/Cargo.toml @@ -43,15 +43,12 @@ countmin = [] cpc = [] frequencies = [] hll = [] -kll = ["dep:rand"] -req = ["dep:rand"] +kll = [] +req = [] tdigest = [] theta = [] tuple = [] -[dependencies] -rand = { workspace = true, optional = true } - [dev-dependencies] googletest = { workspace = true } insta = { workspace = true } diff --git a/datasketches/src/kll/sketch.rs b/datasketches/src/kll/sketch.rs index 82399cdb..38b9245f 100644 --- a/datasketches/src/kll/sketch.rs +++ b/datasketches/src/kll/sketch.rs @@ -43,6 +43,7 @@ use crate::codec::assert::ensure_serial_version_is; use crate::codec::assert::insufficient_data; use crate::codec::family::Family; use crate::common::SearchCriteria; +use crate::common::random::random_bit; use crate::error::Error; /// KLL sketch for estimating quantiles and ranks. @@ -732,7 +733,7 @@ impl KllSketch { current, level, self.is_level_zero_sorted, - rand::random::(), + random_bit(), use_up, ); if above.is_empty() { @@ -985,7 +986,7 @@ fn general_compress( current, current_level, is_level_zero_sorted, - rand::random::(), + random_bit(), use_up, ); let promoted_len = promoted.len(); diff --git a/datasketches/src/req/compactor.rs b/datasketches/src/req/compactor.rs index 8f9cc92d..a56c0fa6 100644 --- a/datasketches/src/req/compactor.rs +++ b/datasketches/src/req/compactor.rs @@ -20,6 +20,7 @@ //! Each level in the REQ sketch uses a compactor to maintain a bounded set of items //! with deterministic compaction when capacity is exceeded. +use crate::common::random::random_bit; use crate::error::Error; use crate::req::INITIAL_SECTIONS_PER_COMPACTOR; use crate::req::MIN_K; @@ -244,7 +245,7 @@ where if (self.state & 1) == 1 { self.coin = !self.coin; // flip coin for odd states } else { - self.coin = rand::random::(); // random coin flip for even states + self.coin = random_bit(); // random coin flip for even states } let odds = self.coin; From 13f3573e01c5936f83d1ef42652e4f330c11bdf1 Mon Sep 17 00:00:00 2001 From: Jeff Chung Date: Wed, 2 Sep 2026 17:22:56 +0800 Subject: [PATCH 3/5] doc: add removing rand crate --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f65b2678..4d19a3ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ All significant changes to this project will be documented in this file. * T-Digest deserialization now rejects unknown or conflicting flags, reversed extrema, out-of-range values, unsorted centroids, and non-empty images without stored values. +### Notable changes + +* The crate no longer has any runtime dependencies. The `kll` and `req` features previously pulled in `rand`; compaction now draws its coin from an in-tree generator. + ## v0.5.0 ### Breaking changes From a6eec04f3f0ee2dbfc464815b02f33d62ce12774 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 2 Sep 2026 21:38:56 +0800 Subject: [PATCH 4/5] docs: clarify random source and changelog category --- CHANGELOG.md | 5 +---- datasketches/src/common/random.rs | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d19a3ff..eaef9798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,15 +15,12 @@ All significant changes to this project will be documented in this file. ### Improvements * Improve truncated-input diagnostics across sketch deserializers. +* The crate no longer has any runtime dependencies. The `kll` and `req` features previously pulled in `rand`; compaction now draws its coin from an in-tree generator. ### Bug fixes * T-Digest deserialization now rejects unknown or conflicting flags, reversed extrema, out-of-range values, unsorted centroids, and non-empty images without stored values. -### Notable changes - -* The crate no longer has any runtime dependencies. The `kll` and `req` features previously pulled in `rand`; compaction now draws its coin from an in-tree generator. - ## v0.5.0 ### Breaking changes diff --git a/datasketches/src/common/random.rs b/datasketches/src/common/random.rs index e8443e35..4d5f41c8 100644 --- a/datasketches/src/common/random.rs +++ b/datasketches/src/common/random.rs @@ -48,9 +48,9 @@ fn next_u64(state: u64) -> (u64, u64) { /// Returns the seed for the calling thread's stream. /// -/// The seed comes from the operating-system entropy that `std` already holds to -/// key its hash maps, mixed with the current thread's id so that threads in one -/// process start at unrelated points of the cycle. +/// The seed is derived from `RandomState`'s randomized hash keys and the current +/// thread's id so that threads in one process start at unrelated points of the +/// cycle. fn random_seed() -> u64 { RandomState::new().hash_one(std::thread::current().id()) } From 07e77d56b925428913c495baed7920c66c6e6a7e Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 2 Sep 2026 21:48:25 +0800 Subject: [PATCH 5/5] refactor: rely on random module visibility --- datasketches/src/common/random.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasketches/src/common/random.rs b/datasketches/src/common/random.rs index 4d5f41c8..9959b58b 100644 --- a/datasketches/src/common/random.rs +++ b/datasketches/src/common/random.rs @@ -66,7 +66,7 @@ thread_local! { /// The call is a handful of arithmetic operations on thread-local state: it /// never allocates, locks, or blocks. The first call on a thread seeds that /// thread's state. -pub(crate) fn random_bit() -> bool { +pub fn random_bit() -> bool { STATE.with(|state| { let (next, value) = next_u64(state.get()); state.set(next);