diff --git a/ci/check_wheel.py b/ci/check_wheel.py index 4a5999e..35f5c4a 100644 --- a/ci/check_wheel.py +++ b/ci/check_wheel.py @@ -5,7 +5,7 @@ Run with ``python -I`` from a clean virtual environment. Covers both table placements (Arrow tables in memory, Lance datasets on disk), CSR traversal, shortestPath, and -``refresh()`` picking up a new Lance version. +``checkout_latest()`` picking up a new Lance version. """ import importlib.metadata @@ -57,17 +57,17 @@ def main(): engine = CypherEngine(config, datasets, use_csr=True) assert engine.execute(TWO_HOPS).to_pylist() == [{"name": "Carol"}] assert engine.execute(SHORTEST).to_pylist() == [{"hops": 3}] - # The last engine reads Lance datasets: refresh() follows an appended version. + # The last engine reads Lance datasets: checkout_latest() follows an appended version. appended = pa.table({"src": [4], "dst": [1]}) lance.write_dataset(appended, Path(root) / "knows.lance", mode="append") - engine.refresh() + engine.checkout_latest() got = engine.execute( "MATCH (a:Person {id: 4})-[:KNOWS]->(b:Person) RETURN b.id AS id" ) assert got.to_pylist() == [{"id": 1}] print( f"Installed lance-csr {importlib.metadata.version('lance-csr')} wheel passed " - f"in-memory, on-disk, CSR, shortestPath and refresh checks on Python " + f"in-memory, on-disk, CSR, shortestPath and checkout_latest checks on Python " f"{sys.version.split()[0]} without Rust" ) diff --git a/crates/lance-csr-python/src/graph.rs b/crates/lance-csr-python/src/graph.rs index 45a1026..0144a1c 100644 --- a/crates/lance-csr-python/src/graph.rs +++ b/crates/lance-csr-python/src/graph.rs @@ -1124,11 +1124,6 @@ impl CypherEngine { .map_err(graph_error_to_pyerr) } - /// Deprecated alias; the Python wrapper warns before calling it. - fn refresh(&self, py: Python) -> PyResult<()> { - self.checkout_latest(py) - } - fn optimize_csr(&self, py: Python) -> PyResult<()> { RT.block_on(Some(py), self.inner.optimize_csr())? .map_err(graph_error_to_pyerr) diff --git a/crates/lance-csr/src/csr_expand.rs b/crates/lance-csr/src/csr_expand.rs index cbf4a7e..62aa08e 100644 --- a/crates/lance-csr/src/csr_expand.rs +++ b/crates/lance-csr/src/csr_expand.rs @@ -107,8 +107,8 @@ use crate::error::{GraphError, Result}; /// /// # Design note for future changes (read before "optimizing" the delta) /// -/// `with_changes` copies the whole delta (not the base) on every refresh, so refresh -/// cost grows with the delta accumulated since the last full build, not just with +/// `with_changes` copies the whole delta (not the base) on every `checkout_latest()`, +/// so its cost grows with the delta accumulated since the last full build, not just with /// the new commit. This is deliberate. Snapshots must stay immutable because /// in-flight queries keep using the previous one, so the delta cannot be mutated in /// place. The copy is bounded: the engine rebuilds once the delta exceeds @@ -117,16 +117,16 @@ use crate::error::{GraphError, Result}; /// occurrence offsets; queries can then resume without allocating deletion maps. /// /// Measured at 10M edges (`python/benchmarks/csr_delta_accumulated.py`, 20 × 1% -/// appends): refresh grows from about 11 ms to about 80 ms as the delta reaches 20%, +/// appends): `checkout_latest()` grows from about 11 ms to about 80 ms as the delta reaches 20%, /// against 1.2–1.6 s for a full rebuild. Two alternatives were tried or assessed /// and rejected: -/// - shared, immutable delta layers merged like a binary counter: median refresh -/// 74 → 32 ms, but merge cascades spiked to 298 ms, lookups got 5–25% slower, and +/// - shared, immutable delta layers merged like a binary counter: median +/// `checkout_latest()` 74 → 32 ms, but merge cascades spiked to 298 ms, lookups got 5–25% slower, and /// the code was about 60 lines larger; /// - a persistent hash map (`imbl`): no spikes and one lookup, but about five new /// crates and `unsafe` code to trust. /// -/// Revisit if refreshes become frequent on graphs far larger than 10M edges, or if +/// Revisit if calls to `checkout_latest()` become frequent on graphs far larger than 10M edges, or if /// the rebuild threshold is raised. pub struct Adjacency { base: Arc, @@ -200,7 +200,7 @@ impl Adjacency { /// Removals apply to the edges present before this call, so a rewritten fragment /// can drop its old rows and add new ones under the same fragment id. `self` is /// unchanged, so snapshots still using it keep their answers. The base is never copied. - // shortcut: copies the accumulated delta on refresh; bounded by the engine's 25% rebuild threshold. Upgrade to a persistent map if refreshes get frequent on much larger graphs. + // shortcut: copies the accumulated delta on each `checkout_latest()`; bounded by the engine's 25% rebuild threshold. Upgrade to a persistent map if `checkout_latest()` gets frequent on much larger graphs. pub fn with_changes(&self, added: &[(i64, i64, u64)], removed: &RemovedRows) -> Self { let mut next = Self { base: self.base.clone(), diff --git a/crates/lance-csr/src/engine.rs b/crates/lance-csr/src/engine.rs index 514e5c1..abd0ee7 100644 --- a/crates/lance-csr/src/engine.rs +++ b/crates/lance-csr/src/engine.rs @@ -318,7 +318,7 @@ struct CsrEntry { base_version: Option, /// Wall time of the last build or delta application. build_ms: f64, - /// How the last refresh or optimize produced this entry: build, rebuild, loaded, delta or reused. + /// How the last build, `checkout_latest()` or optimize produced this entry: build, rebuild, loaded, delta or reused. update: &'static str, } @@ -326,9 +326,10 @@ struct CsrEntry { /// tombstones, accumulated since the last full build) would exceed this fraction /// of the base. Lookups read the delta on every query, and a rebuild clears it. /// -/// This threshold also bounds refresh cost: each delta refresh copies the whole -/// accumulated delta (see the design note on `crate::csr_expand::Adjacency`). -/// Raising it makes refreshes slower as well as lookups. +/// This threshold also bounds the cost of `checkout_latest()`: each delta it applies +/// copies the whole accumulated delta (see the design note on +/// `crate::csr_expand::Adjacency`). +/// Raising it makes `checkout_latest()` slower as well as lookups. // shortcut: fixed threshold, chosen from python/benchmarks/csr_delta_accumulated.py; tune per workload if deltas grow large. const CSR_REBUILD_FRACTION: f64 = 0.25; @@ -741,7 +742,7 @@ fn memtable_partitions(batch: &RecordBatch) -> Vec> { /// source and target counts when the CSR is built, and otherwise one scan of the two /// endpoint columns; only integer endpoints are counted. Reused while a dataset is /// the same snapshot, so an unchanged `checkout_latest()` costs nothing. -// shortcut: the fallback scan reads every edge endpoint at construction and on each changed refresh (about one CSR build's scan); switch to an approximate counter (HyperLogLog) or Lance column statistics if that shows up in startup time. +// shortcut: the fallback scan reads every edge endpoint at construction and on each `checkout_latest()` that finds changes (about one CSR build's scan); switch to an approximate counter (HyperLogLog) or Lance column statistics if that shows up in startup time. async fn build_key_statistics( config: &GraphConfig, sources: &HashMap, @@ -793,7 +794,7 @@ async fn build_key_statistics( dst.name().clone(), )); match csr.get(&mapping.relationship_type) { - // Built from this same snapshot in this refresh, so its counts are current. + // Built from this same snapshot in this `checkout_latest()`, so its counts are current. Some(entry) => { distinct.insert(src.name().clone(), entry.outgoing.num_sources()); distinct.insert(dst.name().clone(), entry.incoming.num_sources()); @@ -923,7 +924,7 @@ async fn build_csr_entries( } } - // The one full build: the first, a refresh whose delta would pass the threshold, + // The one full build: the first, a `checkout_latest()` whose delta would pass the threshold, // and optimize_csr() (`rebuild_deltas`) all end here. let started = std::time::Instant::now(); // Two projected scans of the endpoint columns, one per direction. @@ -1339,12 +1340,6 @@ impl CypherEngine { Ok(()) } - /// Former name of [`checkout_latest`](Self::checkout_latest). - #[deprecated(note = "renamed to `checkout_latest`, after Lance's `Dataset::checkout_latest`")] - pub async fn refresh(&self) -> Result<()> { - self.checkout_latest().await - } - /// Rebuild each CSR that carries a delta, at the snapshot the engine is pinned to: the /// CSR's counterpart of Lance's `optimize_indices()`. /// @@ -1650,7 +1645,7 @@ mod tests { assert_eq!(first, again); assert_eq!(first.num_rows(), 1); - // A refresh publishes a new snapshot with an empty cache; the old one is untouched. + // `checkout_latest()` publishes a new snapshot with an empty cache; the old one is untouched. engine.checkout_latest().await.unwrap(); assert!(engine.snapshot().plans.lock().unwrap().is_empty()); assert_eq!(cached(), PLAN_CACHE_ENTRIES); @@ -1717,7 +1712,7 @@ mod tests { let sources = HashMap::from([("Person".to_string(), source)]); let engine = Arc::new(CypherEngine::new(config, sources, true).await.unwrap()); - // Stand in for refresh A, still building its replacement. + // Stand in for `checkout_latest()` A, still building its replacement. let in_flight = engine.update_lock.lock().await; let b = tokio::spawn({ let engine = engine.clone(); diff --git a/docs/benchmarks.md b/docs/benchmarks.md index ebc825d..55d60fe 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -165,7 +165,7 @@ uv run --no-sync python benchmarks/csr_expand.py --data /tmp/csr-bench --edges 1 | --- | --- | | `csr_expand.py` | CSR build cost, single-hop latency against the join path, `checkout_latest()` cost and peak RSS, per edge count | | `csr_delta.py` | `checkout_latest()` with a delta against a full rebuild, after one append or delete commit (uses `csr_expand.py`'s data) | -| `csr_delta_accumulated.py` | Refresh cost as deltas accumulate over many small commits (the basis of the 25% rebuild threshold) | +| `csr_delta_accumulated.py` | `checkout_latest()` cost as deltas accumulate over many small commits (the basis of the 25% rebuild threshold) | | `engine_cache.py` | Engine construction, first-query, warm-query and `checkout_latest()` timings; can compare against upstream `lance_graph` | | `bounded_memory.py` | Peak memory of streamed scans over Lance data larger than a memory cap (`generate`, then `run`) | diff --git a/docs/internals/execution-plumbing.md b/docs/internals/execution-plumbing.md index fcd5af8..9271d4d 100644 --- a/docs/internals/execution-plumbing.md +++ b/docs/internals/execution-plumbing.md @@ -16,7 +16,7 @@ partition per core. Lance datasets are scanned by Lance's own parallel reader. Each [snapshot](../execution.md#snapshots-and-concurrency) keeps up to 128 unoptimized DataFusion plan templates, least recently used first out, keyed by the query text and the types of its parameters. Running a query again with new scalar `$parameters` skips parsing and graph planning, and only binds the new values. Some operators need literal -values while planning; when a template can't be built, the query is planned in full as before. A refresh starts the +values while planning; when a template can't be built, the query is planned in full as before. `checkout_latest()` starts the new snapshot with an empty cache. CALL queries skip the cache and are planned in full. ## Cooperative deadlines diff --git a/docs/internals/index.md b/docs/internals/index.md index dba91d8..15dfd74 100644 --- a/docs/internals/index.md +++ b/docs/internals/index.md @@ -15,7 +15,7 @@ the data structure those pieces read. | --- | --- | | [Custom operators](operators.md) | `CsrExpand`, `NodeLookup`, `PathExists`, `ShortestPath`, `PathMatch`, `Apply` and `KeyStatistics`, with their plans | | [Optimizer rules and rewrites](optimizer.md) | The two logical optimizer rules, graph rewrites before DataFusion sees the plan, and join statistics | -| [Indexes and refresh](indexes.md) | How the CSR adjacency and node indexes are laid out, and how a refresh diffs Lance versions | +| [Indexes and updates](indexes.md) | How the CSR adjacency and node indexes are laid out, and how `checkout_latest()` diffs Lance versions | | [Execution plumbing](execution-plumbing.md) | Partitioning, the plan template cache, deadlines and Cypher functions | ## What lance-csr added to lance-graph @@ -34,7 +34,7 @@ Added in lance-csr : Everything this section describes as a DataFusion extension: the extension nodes and their operators, the extension planner, both optimizer rules, the deadline rule and the join statistics. Also the graph rewrites (re-rooting, alternation and multi-label branching), the `OptionalMatch`, `ShortestPath`, `PathMatch` and `Apply` - operators, the CSR adjacency and node indexes with incremental refresh, engine snapshots, the plan template cache + operators, the CSR adjacency and node indexes with incremental updates, engine snapshots, the plan template cache and the Cypher-specific functions. [lance-graph](https://github.com/lance-format/lance-graph) plans every hop as joins, and lance-csr keeps that planning @@ -109,7 +109,7 @@ All paths are under | File | What it holds | Origin | | --- | --- | --- | -| `csr_expand.rs` | `Adjacency`, `NodeRows`, `EdgeRows`, `CsrExpandNode` and `CsrExpandExec`, `CsrExtensionPlanner`, `CsrQueryPlanner`, both optimizer rules, fragment diffing for refresh | lance-csr | +| `csr_expand.rs` | `Adjacency`, `NodeRows`, `EdgeRows`, `CsrExpandNode` and `CsrExpandExec`, `CsrExtensionPlanner`, `CsrQueryPlanner`, both optimizer rules, fragment diffing for `checkout_latest()` | lance-csr | | `csr_expand/node_lookup.rs` | `NodeLookupNode` and `NodeLookupExec` | lance-csr | | `csr_index.rs` | The compact CSR neighbour store | [lance-graph](https://github.com/lance-format/lance-graph); edge rows and the delta added in lance-csr, unused traversal helpers removed | | `csr_traversal.rs` | Resumable, typed variable-length walks | lance-csr | @@ -120,6 +120,6 @@ All paths are under | `pattern_order.rs`, `pattern_alternatives.rs` | Re-rooting, predicates implied by an `OR`, alternation and multi-label branching | lance-csr | | `logical_plan.rs` | Graph logical operators | [lance-graph](https://github.com/lance-format/lance-graph), extended in lance-csr | | `datafusion_planner/` | Graph plan to DataFusion plan; `builder/expand_ops.rs` chooses CSR or join for each hop | [lance-graph](https://github.com/lance-format/lance-graph), extended in lance-csr | -| `engine.rs` | `CypherEngine`: snapshots, session setup, refresh, plan template cache, SQL entry points | lance-csr | +| `engine.rs` | `CypherEngine`: snapshots, session setup, `checkout_latest()`, plan template cache, SQL entry points | lance-csr | | `graph_table.rs` | The `GRAPH_TABLE` rewrite and `GraphTableFunction` | lance-csr | | `query/deadline.rs` | `DeadlineJoins` and `with_execution_deadline` | lance-csr | diff --git a/docs/internals/indexes.md b/docs/internals/indexes.md index 43aa2d9..ecbdc09 100644 --- a/docs/internals/indexes.md +++ b/docs/internals/indexes.md @@ -1,4 +1,4 @@ -# Indexes and refresh +# Indexes and updates The [custom operators](operators.md) read two in-memory indexes. Both are built from the same underlying Lance dataset and point back into it, so neither holds a copy of node or relationship properties. [The CSR index](../csr.md) covers what users see: @@ -7,7 +7,7 @@ are laid out and maintained. The compact store of offsets and neighbours came from [lance-graph](https://github.com/lance-format/lance-graph), whose query planner didn't use it. lance-csr added the edge rows, the delta and tombstones, the node index and -incremental refresh, and wired the store into query execution through the [custom operators](operators.md). +incremental updates, and wired the store into query execution through the [custom operators](operators.md). ```text Lance datasets (pinned version) @@ -39,7 +39,7 @@ each type has an outgoing and an incoming one. It is built from the relationship - each neighbour also records the row its edge came from: its Lance row address (fragment id in the high 32 bits, offset in the low 32), or its row number for in-memory input. -The edge row is what lets a hop fetch relationship properties with `take`, and what lets a refresh find deleted edges +The edge row is what lets a hop fetch relationship properties with `take`, and what lets `checkout_latest()` find deleted edges without reading the old dataset version. Edges with a null endpoint are skipped: they could never satisfy the equality join the CSR replaces. @@ -51,7 +51,7 @@ from that minimum. Neighbouring slots mostly come from nearby rows of the same f GDC SF1 edge rows take about 2 bytes per edge in each direction instead of 8, which cut the CSR's arrays from 613 to 419 MiB. Any position stays directly readable, which traversal needs to resume part-way through a list, and reading a range loads each block's header once. Plain -hops never unpack edge rows; only hops that name their relationship, and refresh deletes, do. +hops never unpack edge rows; only hops that name their relationship, and deletes applied by `checkout_latest()`, do. Neighbours stay a plain `u64` array. Packing them the same way roughly halved them on SF1 (their keys are sparse: Comment keys span 2.2 trillion for 2 million rows, so a single width for the whole array, as a `u32` offset, wouldn't @@ -65,7 +65,7 @@ by scanning only the id column. A label whose ids aren't integers, or aren't uni matches several rows in a join, which a single position can't reproduce. The node index depends only on its node table, so edge commits update only the adjacency and node commits rebuild only the node index. -## How a refresh diffs Lance versions +## How `checkout_latest()` diffs Lance versions { #how-checkout-latest-diffs-lance-versions } `checkout_latest()` compares the pinned and latest versions of each relationship dataset fragment by fragment, from their manifests: @@ -86,25 +86,25 @@ threshold. ## The delta -The base CSR is immutable and shared between snapshots through an `Arc`. A refresh applies changes as a delta on top +The base CSR is immutable and shared between snapshots through an `Arc`. `checkout_latest()` applies changes as a delta on top of it: appended edges per source key, and tombstones stored as sorted offsets into each source's neighbours. Offsets let readers skip deleted occurrences without copying deletion state for each query. Identical edges stay distinct occurrences. -Each refresh copies the whole delta, never the base. Snapshots must stay immutable because running queries keep using +Each `checkout_latest()` call copies the whole delta, never the base. Snapshots must stay immutable because running queries keep using the previous one, so the delta can't be mutated in place. The copy is bounded because the engine rebuilds once the delta passes 25% of the base. Measured at 10 million edges with `python/benchmarks/csr_delta_accumulated.py` (twenty -appends of 1% each), a refresh grows from about 11 ms to about 80 ms as the delta reaches 20%, against 1.2 to 1.6 s +appends of 1% each), a call grows from about 11 ms to about 80 ms as the delta reaches 20%, against 1.2 to 1.6 s for a full rebuild. ??? note "Alternatives that were tried and rejected" - - **Shared, immutable delta layers merged like a binary counter.** Median refresh fell from 74 to 32 ms, but merge + - **Shared, immutable delta layers merged like a binary counter.** The median call fell from 74 to 32 ms, but merge cascades spiked to 298 ms, lookups got 5 to 25% slower, and the code was about 60 lines larger. - **A persistent hash map (`imbl`).** No spikes and one lookup, but about five new crates and `unsafe` code to trust. - Revisit this if refreshes become frequent on graphs far larger than 10 million edges, or if the rebuild threshold + Revisit this if calls to `checkout_latest()` become frequent on graphs far larger than 10 million edges, or if the rebuild threshold is raised. The base and delta map onto Lance's own index design: the delta plays the part of Lance's unindexed rows, and @@ -130,8 +130,8 @@ The arrays only make sense to `csr_store.rs`, which validates every one before u the raw parts, not neighbour lists. The schema metadata records the format, the dataset version, the endpoint column names, their Lance field ids and types, and the fragments the base was built from. The delta is never stored. -When an engine starts, it reads the newest file at or before the pinned version. It then diffs that file's fragments against the pinned version, as a refresh diffs two versions (see -[How a refresh diffs Lance versions](#how-a-refresh-diffs-lance-versions)), so the file's own version doesn't need to +When an engine starts, it reads the newest file at or before the pinned version. It then diffs that file's fragments against the pinned version, as `checkout_latest()` diffs two versions (see +[How `checkout_latest()` diffs Lance versions](#how-checkout-latest-diffs-lance-versions)), so the file's own version doesn't need to exist any more. An empty diff uses the file as it is; otherwise the changes become the delta, under the same 25% threshold. Rebuilds after start-up (from `checkout_latest()` or `optimize_csr()`) don't look for a file, since the stored base is the one the engine already holds, but they do store the new one. diff --git a/docs/lance.md b/docs/lance.md index dbe8fe6..f303b58 100644 --- a/docs/lance.md +++ b/docs/lance.md @@ -67,11 +67,6 @@ print(engine.execute("MATCH (p:Person) RETURN count(*) AS n").to_pylist()) # [{ print(engine.dataset_versions()) # {'Person': 2, 'KNOWS': 2} ``` -!!! note "Formerly `refresh()`" - - This method was called `refresh()` before. `refresh()` still works, and does exactly the same, but it - warns with `DeprecationWarning` and will be removed in a later release. - What `checkout_latest()` guarantees: - **Unchanged datasets keep their handle.** A dataset whose version and commit time are unchanged is reused, diff --git a/python/benchmarks/csr_delta.py b/python/benchmarks/csr_delta.py index 427bab7..28a8f9c 100644 --- a/python/benchmarks/csr_delta.py +++ b/python/benchmarks/csr_delta.py @@ -116,7 +116,7 @@ def main() -> None: engine = CypherEngine(CONFIG, datasets, use_csr=True) change(uri, nodes) - refresh_ms, _ = timed(engine.checkout_latest) + checkout_ms, _ = timed(engine.checkout_latest) stats = engine.csr_stats()["KNOWS"] latest = {"Person": person, "KNOWS": lance.dataset(uri)} @@ -128,7 +128,7 @@ def main() -> None: "update": stats["update"], "delta_edges": stats["delta_edges"], "tombstones": stats["tombstones"], - "refresh_ms": refresh_ms, + "checkout_ms": checkout_ms, "delta_apply_ms": stats["build_ms"], "rebuild_ms": fresh.csr_stats()["KNOWS"]["build_ms"], } diff --git a/python/benchmarks/csr_delta_accumulated.py b/python/benchmarks/csr_delta_accumulated.py index 5dbfac2..b7f789f 100644 --- a/python/benchmarks/csr_delta_accumulated.py +++ b/python/benchmarks/csr_delta_accumulated.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors -"""Refresh cost as CSR deltas accumulate over many small commits. +"""`checkout_latest()` cost as CSR deltas accumulate over many small commits. Uses a graph written by ``csr_expand.py``. On a fresh copy of the edge table, appends ``--step`` of the edges ``--commits`` times, calling ``checkout_latest()`` after -each, and prints each refresh's time with the delta size so far. With deltas that -are copied on every refresh, the time grows with the accumulated delta; with +each, and prints each call's time with the delta size so far. With deltas that +are copied on every call, the time grows with the accumulated delta; with shared delta layers it should stay roughly proportional to the step. python benchmarks/csr_delta_accumulated.py --data /tmp/csr-bench --edges 10000000 @@ -60,7 +60,7 @@ def main() -> None: lance.write_dataset(new, uri, mode="append") start = time.perf_counter() engine.checkout_latest() - refresh_ms = (time.perf_counter() - start) * 1e3 + checkout_ms = (time.perf_counter() - start) * 1e3 stats = engine.csr_stats()["KNOWS"] print( json.dumps( @@ -68,7 +68,7 @@ def main() -> None: "commit": commit + 1, "update": stats["update"], "delta_edges": stats["delta_edges"], - "refresh_ms": refresh_ms, + "checkout_ms": checkout_ms, } ), flush=True, diff --git a/python/benchmarks/csr_expand.py b/python/benchmarks/csr_expand.py index 5134b51..02cdeb9 100644 --- a/python/benchmarks/csr_expand.py +++ b/python/benchmarks/csr_expand.py @@ -9,7 +9,7 @@ - engine construction time, and for ``csr`` the CSR build time (both directions); - warm latency (median/p95) of single-hop queries, with row counts and checksums checked against the other mode; -- refresh cost after appending edges (CSR rebuild) and with nothing changed; +- ``checkout_latest()`` cost after appending edges (CSR rebuild) and with nothing changed; - peak RSS of the process. python benchmarks/csr_expand.py --data /tmp/csr-bench --edges 100000 1000000 @@ -125,11 +125,11 @@ def open_engine(): t = time.perf_counter() engine.checkout_latest() - out["refresh_unchanged_ms"] = (time.perf_counter() - t) * 1e3 + out["checkout_unchanged_ms"] = (time.perf_counter() - t) * 1e3 # Recorded before the copy below, which materializes the edge table. out["peak_rss_mib"] = peak_rss_mib() - # Append 0.1% more edges, then refresh: the CSR engine must rebuild. + # Append 0.1% more edges, then check out the latest version: the CSR engine must rebuild. n_new = max(1, lance.dataset(root / "knows.lance").count_rows() // 1000) new_edges = pa.table( { @@ -150,7 +150,7 @@ def open_engine(): lance.write_dataset(new_edges, knows, mode="append") t = time.perf_counter() engine.checkout_latest() - out["refresh_after_append_ms"] = (time.perf_counter() - t) * 1e3 + out["checkout_after_append_ms"] = (time.perf_counter() - t) * 1e3 if mode == "csr": out["csr_rebuild_ms"] = engine.csr_stats()["KNOWS"]["build_ms"] return out diff --git a/python/benchmarks/engine_cache.py b/python/benchmarks/engine_cache.py index ab32a06..f6afad2 100644 --- a/python/benchmarks/engine_cache.py +++ b/python/benchmarks/engine_cache.py @@ -112,17 +112,16 @@ def emit(case, **values): warm.append(t) emit("warm_query", **summary(warm)) - # Upstream has no checkout_latest() (refresh() in older lance-csr); its equivalent - # is building a new engine. - refresh = [] - checkout = getattr(engine, "checkout_latest", None) or getattr(engine, "refresh", None) + # Upstream has no checkout_latest(); its equivalent is building a new engine. + checkouts = [] + checkout = getattr(engine, "checkout_latest", None) for _ in range(min(args.repeats, 10)): if checkout is not None: t, _ = timed(checkout) else: t, engine = timed(lambda: mod.CypherEngine(config, open_datasets())) - refresh.append(t) - emit("refresh_unchanged", **summary(refresh)) + checkouts.append(t) + emit("checkout_unchanged", **summary(checkouts)) # Control: no engine, inputs converted and catalog rebuilt for every query. rebuild = [] diff --git a/python/python/lance_csr/_internal/graph.pyi b/python/python/lance_csr/_internal/graph.pyi index 151ac0d..da448b9 100644 --- a/python/python/lance_csr/_internal/graph.pyi +++ b/python/python/lance_csr/_internal/graph.pyi @@ -134,7 +134,6 @@ class CypherEngine: self, query: str, params: dict[str, _ParameterT] | None = None ) -> str: ... def checkout_latest(self) -> None: ... - def refresh(self) -> None: ... def optimize_csr(self) -> None: ... def delete_stored_csr(self) -> int: ... def node_index_stats(self) -> dict[str, _NodeIndexStats]: ... diff --git a/python/python/lance_csr/cypher.py b/python/python/lance_csr/cypher.py index 504f4df..0a7440c 100644 --- a/python/python/lance_csr/cypher.py +++ b/python/python/lance_csr/cypher.py @@ -2,7 +2,6 @@ from __future__ import annotations -import warnings from enum import IntEnum from typing import TYPE_CHECKING, Any, Self, final @@ -273,21 +272,6 @@ def checkout_latest(self) -> None: """ self._inner.checkout_latest() - def refresh(self) -> None: - """Deprecated alias of `checkout_latest`. - - Behaves exactly like - [`checkout_latest`][lance_csr.CypherEngine.checkout_latest], and warns with - `DeprecationWarning`. - """ - warnings.warn( - "CypherEngine.refresh() is deprecated; use checkout_latest(), named after " - "Lance's LanceDataset.checkout_latest()", - DeprecationWarning, - stacklevel=2, - ) - self.checkout_latest() - def optimize_csr(self) -> None: """Rebuild each CSR that carries a delta (experimental; needs `use_csr=True`). diff --git a/python/python/tests/test_call_subqueries.py b/python/python/tests/test_call_subqueries.py index 454dfe8..91343a9 100644 --- a/python/python/tests/test_call_subqueries.py +++ b/python/python/tests/test_call_subqueries.py @@ -396,7 +396,7 @@ def test_rich_path_parameters_bind_freshly(graph): assert engine.execute(query, {"start": start}).column(0).to_pylist() == expected -def test_rich_path_reader_retains_snapshot_after_refresh(graph, tmp_path): +def test_rich_path_reader_retains_snapshot_after_checkout_latest(graph, tmp_path): import lance original, config, data = graph diff --git a/python/python/tests/test_engine_lance.py b/python/python/tests/test_engine_lance.py index 7c14962..9cd5dc8 100644 --- a/python/python/tests/test_engine_lance.py +++ b/python/python/tests/test_engine_lance.py @@ -102,7 +102,7 @@ def test_engine_empty_lance_dataset(tmp_path): assert count_people(engine) == 0 -def test_snapshot_until_refresh_after_append(graph): +def test_snapshot_until_checkout_latest_after_append(graph): path, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) @@ -114,16 +114,7 @@ def test_snapshot_until_refresh_after_append(graph): assert engine.dataset_versions()["Person"] == 2 -def test_refresh_is_a_deprecated_alias_of_checkout_latest(graph): - path, person, knows = graph - engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) - lance.write_dataset(people(N, N + 5), path / "person.lance", mode="append") - with pytest.warns(DeprecationWarning, match="checkout_latest"): - engine.refresh() - assert count_people(engine) == N + 5 - - -def test_refresh_without_changes_keeps_results(graph): +def test_checkout_latest_without_changes_keeps_results(graph): _, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) before = engine.execute("MATCH (p:Person) WHERE p.id < 3 RETURN p.name") @@ -133,7 +124,7 @@ def test_refresh_without_changes_keeps_results(graph): assert engine.dataset_versions() == {"Person": 1, "KNOWS": 1} -def test_refresh_sees_overwrite_with_added_and_retyped_columns(graph): +def test_checkout_latest_sees_overwrite_with_added_and_retyped_columns(graph): path, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) with pytest.raises(Exception): @@ -151,7 +142,7 @@ def test_refresh_sees_overwrite_with_added_and_retyped_columns(graph): assert sorted(out.column(1).to_pylist()) == [30, 40] -def test_refresh_sees_table_replaced_at_same_location(graph): +def test_checkout_latest_sees_table_replaced_at_same_location(graph): path, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) @@ -163,7 +154,7 @@ def test_refresh_sees_table_replaced_at_same_location(graph): @pytest.mark.parametrize("use_csr", [False, True]) -def test_failed_refresh_keeps_previous_snapshot(graph, use_csr): +def test_failed_checkout_latest_keeps_previous_snapshot(graph, use_csr): path, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}, use_csr=use_csr) @@ -207,14 +198,14 @@ class Wrapped(lance.LanceDataset): person.__class__ = Wrapped engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) - # Treated as Lance (pinned, refreshable), not materialized as Arrow. + # Treated as Lance (pinned, moved by checkout_latest()), not materialized as Arrow. assert engine.dataset_versions()["Person"] == 1 lance.write_dataset(people(N, N + 1), path / "person.lance", mode="append") engine.checkout_latest() assert count_people(engine) == N + 1 -def test_concurrent_execute_and_refresh(graph): +def test_concurrent_execute_and_checkout_latest(graph): path, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}) allowed = {N + 5 * i for i in range(6)} @@ -266,7 +257,7 @@ def test_execute_batches_partial_consumption(graph): @pytest.mark.parametrize("use_csr", [False, True]) -def test_execute_batches_keeps_snapshot_across_refresh(graph, use_csr): +def test_execute_batches_keeps_snapshot_across_checkout_latest(graph, use_csr): path, person, knows = graph engine = CypherEngine(CONFIG, {"Person": person, "KNOWS": knows}, use_csr=use_csr) reader = engine.execute_batches("MATCH (p:Person) RETURN p.id") diff --git a/python/python/tests/test_path_values.py b/python/python/tests/test_path_values.py index c21e391..c8b51b8 100644 --- a/python/python/tests/test_path_values.py +++ b/python/python/tests/test_path_values.py @@ -578,7 +578,9 @@ def test_cross_label_shortest_scopes_rebind_each_parameter(cross_label_graph, sc assert engine.execute_batches(query, params).read_all().to_pylist() == expected -def test_cross_label_shortest_inventory_isolates_refresh(cross_label_graph, tmp_path): +def test_cross_label_shortest_inventory_isolates_checkout_latest( + cross_label_graph, tmp_path +): import lance config, tables, use_csr = cross_label_graph diff --git a/python/python/tests/test_target_partitions.py b/python/python/tests/test_target_partitions.py index 676ad61..133a3e8 100644 --- a/python/python/tests/test_target_partitions.py +++ b/python/python/tests/test_target_partitions.py @@ -28,7 +28,7 @@ def graph(): @pytest.mark.parametrize("partitions", [None, 1, 2]) @pytest.mark.parametrize("use_csr", [False, True]) @pytest.mark.parametrize("storage", ["arrow", "lance"]) -def test_partition_targets_preserve_queries_and_refresh( +def test_partition_targets_preserve_queries_and_checkout_latest( graph, partitions, use_csr, storage, tmp_path ): config, data = graph diff --git a/zensical.toml b/zensical.toml index 54e6da2..34ddd9b 100644 --- a/zensical.toml +++ b/zensical.toml @@ -42,7 +42,7 @@ nav = [ "internals/index.md", { "Custom operators" = "internals/operators.md" }, { "Optimizer rules and rewrites" = "internals/optimizer.md" }, - { "Indexes and refresh" = "internals/indexes.md" }, + { "Indexes and updates" = "internals/indexes.md" }, { "Execution plumbing" = "internals/execution-plumbing.md" }, ] }, ] },