Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,15 @@ All significant changes to this project will be documented in this file.

## Unreleased


### Breaking changes

* `BloomFilter::invert` now consumes `self` and returns a read-only `BloomFilterInvertedView` instead of mutating in place. Call `invert()` on the view to restore the original updatable filter.
* Move `SearchCriteria` from `req` to `common` and remove its `Default` implementation. Import `datasketches::common::SearchCriteria` and explicitly choose `Inclusive` or `Exclusive` for each query.

### New features

* Add `BloomFilterInvertedView` to represent inverted Bloom filters with read-only query semantics.
* Add KLL sketches behind the `kll` feature, with rank, quantile, PMF, and CDF queries, merging, totally ordered custom item types, a `KllFloat` adapter for non-NaN floating-point values, and serialization.

### Improvements
Expand Down
11 changes: 7 additions & 4 deletions datasketches/src/bloom/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@
//! * **Fixed size**: Unlike typical sketches, Bloom filters do not resize automatically
//! * **Linear space**: Size is proportional to the expected number of distinct items
//!
//! These guarantees describe normal operation. After [`invert()`](BloomFilter::invert) neither
//! the no-false-negative nor the false-positive guarantee holds; see its documentation.
//! These guarantees describe normal operation. When converted into a [`BloomFilterInvertedView`]
//! via [`invert()`](BloomFilter::invert), neither the no-false-negative nor the false-positive
//! guarantee holds; see its documentation.
//!
//! # Usage
//!
Expand Down Expand Up @@ -128,8 +129,9 @@
//! // Intersect: recognizes only items in both filters
//! // filter1.intersect(&filter2).unwrap();
//!
//! // Invert: approximately inverts set membership
//! // filter1.invert();
//! // Invert: returns a read-only inverted view
//! let inverted = filter1.invert();
//! assert!(!inverted.contains(&"a"));
//! ```
//!
//! # Implementation Details
Expand All @@ -149,3 +151,4 @@ mod sketch;

pub use self::sketch::BloomFilter;
pub use self::sketch::BloomFilterBuilder;
pub use self::sketch::BloomFilterInvertedView;
116 changes: 100 additions & 16 deletions datasketches/src/bloom/sketch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,6 @@ const EMPTY_FLAG_MASK: u8 = 1 << 2;
/// * No false negatives (inserted items always return `true`)
/// * Tunable false positive rate
/// * Constant space usage
///
/// These guarantees hold until [`invert()`](Self::invert) is called; see its documentation.
#[derive(Debug, Clone, PartialEq)]
pub struct BloomFilter {
/// Hash seed for all hash functions
Expand Down Expand Up @@ -251,13 +249,7 @@ impl BloomFilter {
Ok(())
}

/// Inverts all bits in the filter.
///
/// This approximately inverts the notion of set membership. After inversion, neither the
/// no-false-negative nor the false-positive guarantee holds: inserted items may return
/// `false` from [`contains()`](Self::contains), and [`is_empty()`](Self::is_empty),
/// [`bits_used()`](Self::bits_used), and [`load_factor()`](Self::load_factor) describe the
/// raw bit state rather than the inserted items.
/// Consumes the filter and inverts all its bits, returning a read-only inverted view.
///
/// # Examples
///
Expand All @@ -269,20 +261,18 @@ impl BloomFilter {
/// .unwrap();
/// filter.insert("apple");
///
/// filter.invert();
/// // Now "apple" probably returns false, and most other items return true
/// let inverted = filter.invert();
/// assert!(!inverted.contains(&"apple"));
/// ```
pub fn invert(&mut self) {
pub fn invert(mut self) -> BloomFilterInvertedView {
for word in &mut self.bit_array {
*word = !*word;
}
self.num_bits_set = self.capacity() as u64 - self.num_bits_set;
BloomFilterInvertedView { inner: self }
}

/// Returns whether no bits are set in the filter.
///
/// In normal operation this means no items were inserted. After [`invert()`](Self::invert),
/// it reports the raw bit state instead.
/// Returns `true` if no bits are set in the filter.
pub fn is_empty(&self) -> bool {
self.num_bits_set == 0
}
Expand Down Expand Up @@ -613,6 +603,100 @@ impl BloomFilter {
}
}

/// A read-only inverted view of a [`BloomFilter`].
///
/// Created by calling [`BloomFilter::invert()`]. Set membership queries on this view are inverted:
/// a `true` result from [`contains()`](Self::contains) guarantees that the item was definitely not
/// inserted into the filter prior to inversion.
///
/// Modifying operations are omitted to prevent unsound updates. Calling
/// [`invert()`](Self::invert) restores the original [`BloomFilter`].
#[derive(Debug, Clone, PartialEq)]
pub struct BloomFilterInvertedView {
inner: BloomFilter,
}

impl BloomFilterInvertedView {
/// Returns `true` if an item was definitely not in the set prior to inversion.
///
/// # Examples
///
/// ```
/// use datasketches::bloom::BloomFilterBuilder;
///
/// let mut filter = BloomFilterBuilder::with_accuracy(100, 0.01)
/// .build()
/// .unwrap();
/// filter.insert("apple");
///
/// let inverted = filter.invert();
/// assert!(!inverted.contains(&"apple"));
/// ```
pub fn contains<T: Hash>(&self, item: &T) -> bool {
self.inner.contains(item)
}

/// Inverts the view back into an updatable [`BloomFilter`].
///
/// Inverting twice restores the original bit state and filter guarantees.
///
/// # Examples
///
/// ```
/// use datasketches::bloom::BloomFilterBuilder;
///
/// let mut filter = BloomFilterBuilder::with_accuracy(100, 0.01)
/// .build()
/// .unwrap();
/// filter.insert("apple");
///
/// let inverted = filter.invert();
/// let restored = inverted.invert();
/// assert!(restored.contains(&"apple"));
/// ```
pub fn invert(mut self) -> BloomFilter {
for word in &mut self.inner.bit_array {
*word = !*word;
}
self.inner.num_bits_set = self.inner.capacity() as u64 - self.inner.num_bits_set;
self.inner
}

/// Returns whether no bits are set in the inverted filter view.
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}

/// Returns the number of bits set to 1 in the inverted filter.
pub fn bits_used(&self) -> u64 {
self.inner.bits_used()
}

/// Returns the total bit capacity of the filter.
pub fn capacity(&self) -> usize {
self.inner.capacity()
}

/// Returns the number of hash functions used.
pub fn num_hashes(&self) -> u16 {
self.inner.num_hashes()
}

/// Returns the hash seed.
pub fn seed(&self) -> u64 {
self.inner.seed()
}

/// Returns the current load factor of the inverted filter.
pub fn load_factor(&self) -> f64 {
self.inner.load_factor()
}

/// Returns the estimated size of the filter view in bytes.
pub fn estimated_size(&self) -> usize {
self.inner.estimated_size()
}
}
/// Builder for creating [`BloomFilter`] instances.
///
/// Provides two construction modes:
Expand Down
26 changes: 22 additions & 4 deletions tests-integration/tests/bloom_test/sketch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,29 @@ fn test_invert_is_reversible() {

let original = filter.clone();
let original_bits = filter.bits_used();
filter.invert();
assert_eq!(filter.bits_used(), filter.capacity() as u64 - original_bits);

filter.invert();
assert_eq!(filter, original);
let inverted = filter.invert();
assert_eq!(
inverted.bits_used(),
inverted.capacity() as u64 - original_bits
);
assert_eq!(inverted.capacity(), original.capacity());
assert_eq!(inverted.num_hashes(), original.num_hashes());
assert_eq!(inverted.seed(), original.seed());

let restored = inverted.invert();
assert_eq!(restored, original);
}

#[test]
fn test_inverted_view_queries() {
let mut filter = filter();
filter.insert("apple");

let inverted = filter.invert();
assert!(!inverted.contains(&"apple"));
assert_that!(inverted.load_factor(), gt(0.0));
assert_that!(inverted.estimated_size(), gt(0));
}

#[test]
Expand Down