From 3d0ba4b69d6bd60ba844b72d506252d3699bfd08 Mon Sep 17 00:00:00 2001 From: Marius S <39998+winding-lines@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:00:52 -0800 Subject: [PATCH] morca-sdk: replaces= (overwrite a partition) for Rust pipelines (#537) step(..).replaces([..]) mirrors Mojo's replaces=: append and append_files to such a dataset commit the run's first write to each partition as one overwrite snapshot that removes what earlier runs left there, with the orca.replaced-* summary keys, the same fence, find_commit and planning on every retry. iceberg-rust 0.11.0 has no overwrite transaction, so the snapshot is written from iceberg-rust's manifest and manifest-list writers and committed through MorcaCatalog::commit_next. Also write_files/append_files and replace_partitions on Ctx, and the conformance suite's tables.replace* checks, run on the Rust example and its Mojo twin (tests/sensors/replace.mojo), with Mojo's morca.tables as a reader of the replaced table. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017TycszSbyw92Efu4ZeCYL6 --- changes/537-sdk-replaces.minor.md | 1 + docs/pipeline-protocol.md | 28 ++ docs/rust-pipelines.md | 39 ++- pixi.toml | 5 +- rust/morca-sdk/Cargo.toml | 4 + rust/morca-sdk/README.md | 6 +- rust/morca-sdk/examples/replace.rs | 117 ++++++++ rust/morca-sdk/src/ctx.rs | 17 ++ rust/morca-sdk/src/exec.rs | 3 +- rust/morca-sdk/src/pipeline.rs | 68 ++++- rust/morca-sdk/src/tables/catalog.rs | 67 +++-- rust/morca-sdk/src/tables/imp.rs | 210 +++++++++++++-- rust/morca-sdk/src/tables/mod.rs | 8 +- rust/morca-sdk/src/tables/replace.rs | 388 +++++++++++++++++++++++++++ tests/conformance.py | 175 +++++++++++- tests/conformance.sh | 9 +- tests/conformance/replace-rust.json | 18 ++ tests/conformance/replace.json | 15 ++ tests/rust_sdk.sh | 15 +- tests/sensors/replace.mojo | 78 ++++++ tests/sensors/rows_by.mojo | 32 +++ 21 files changed, 1232 insertions(+), 71 deletions(-) create mode 100644 changes/537-sdk-replaces.minor.md create mode 100644 rust/morca-sdk/examples/replace.rs create mode 100644 rust/morca-sdk/src/tables/replace.rs create mode 100644 tests/conformance/replace-rust.json create mode 100644 tests/conformance/replace.json create mode 100644 tests/sensors/replace.mojo create mode 100644 tests/sensors/rows_by.mojo diff --git a/changes/537-sdk-replaces.minor.md b/changes/537-sdk-replaces.minor.md new file mode 100644 index 00000000..ce91c038 --- /dev/null +++ b/changes/537-sdk-replaces.minor.md @@ -0,0 +1 @@ +- **morca-sdk `replaces`**: a Rust step can replace the partitions it writes, as Mojo's `replaces=` does. (#537) diff --git a/docs/pipeline-protocol.md b/docs/pipeline-protocol.md index 0bd1d089..1ae01123 100644 --- a/docs/pipeline-protocol.md +++ b/docs/pipeline-protocol.md @@ -852,6 +852,34 @@ The run token is what keeps two databases over the same tables, or a database restored from a backup, from taking each other's commits for their own (#292). +### Replacing partitions + +A dataset in a step's `replaces` (describe's `replaces`, also in its +`writes`) is one whose partitions a new run replaces rather than appends to +(`replace_partitions`, `src/morca/_lake/impl.mojo:1051-1325`): + +- The run's first commit to a partition is one `overwrite` snapshot that + removes every data and delete file in it that this run didn't commit, and + adds the new files. A file is this run's if its snapshot's summary has + this run's `orca.pipeline` and `orca.run` (`Ctx.same_run`), so later + commits of the run to the partition, from any step or attempt, find only + its own files and append. A commit that removes nothing is an `append`. +- The partitions are those of the new files under the table's current + spec; files under an older spec are left alone, and an unpartitioned + table is one partition. +- What to remove is planned again on every try, against the snapshot the + try commits on top of. +- A file in a partition being replaced that another writer committed after + this run's first commit to the table fails the attempt, committing + nothing. +- The snapshot carries `orca.replaced-files` and `orca.replaced-records`, + and is found again on a retry or resume as any commit is + ([Finding an earlier attempt's commit](#finding-an-earlier-attempts-commit)). + +**Contract** that a rerun of a key leaves one copy of its rows: `morca +backfill --rerun` relies on it, and refuses a pipeline that appends +instead (`appending_writes`, `src/morca/_internal/queueing.mojo:356-366`). + ### Pinned reads Every attempt of a step reads the snapshots the first attempt read, so a diff --git a/docs/rust-pipelines.md b/docs/rust-pipelines.md index a45aedcd..5b3e0415 100644 --- a/docs/rust-pipelines.md +++ b/docs/rust-pipelines.md @@ -94,7 +94,8 @@ it was stored as, so a step whose types changed fails with what it got rather than misreading it. `step("name")` sets a step's `retries`, `backoff_seconds` (doubling), -`timeout_seconds`, data `version`, and its `reads` and `writes`. A plain +`timeout_seconds`, data `version`, and its `reads`, `writes` and +`replaces`. A plain `"name"` takes the defaults: no retries, no timeout. A step fails by returning an error: `?` on any error type, or @@ -169,8 +170,42 @@ Pipeline::::new("clean_rust") writes nothing. `find_commit(dataset)` asks first, for a step that writes files itself. `arrow_schema(dataset)` is the schema the batches need. - `ensure_table(dataset, schema, spec)` creates a table that isn't there. +- `write_files(dataset, batches)` writes the Parquet files without + committing them, and `append_files(dataset, files)` commits them as + `append` would; a step that does this calls `find_commit` first. - A step may read and write only the datasets it declares. +#### Replacing partitions + +A dataset in `replaces` instead of `writes` is one whose partitions a new +run replaces, as Mojo's `replaces=`: `append` and `append_files` to it make +the run's first commit to each partition an `overwrite` that removes what +earlier runs left there, so `morca backfill --rerun` leaves one copy of a +key's rows. + +```rust +.mapped(step("load").reads(["raw.events"]).replaces(["lake.monthly"]), load) +``` + +The semantics are the Mojo side's (docs/pipeline-protocol.md, "Replacing +partitions"): the partitions are those the new rows land in, files this run +committed are never removed, a commit that removes nothing appends, the +plan is made again on every retry, another writer's file in the partition +committed after the run's first commit fails the attempt, and the snapshot +carries `orca.replaced-files` and `orca.replaced-records`. A crash between +writing and committing, and a retry after committing, behave as for an +append. `replace_partitions` and `replace_partition_files` replace a +dataset declared in `writes`. A dataset in `replaces` is hashed as a write, +so declaring it doesn't change the registry hash. + +iceberg-rust 0.11.0 has no overwrite transaction, so morca-sdk writes that +snapshot itself with iceberg-rust's manifest and manifest-list writers +(src/tables/replace.rs) and commits it through its catalog, fenced like any +commit. [examples/replace.rs](../rust/morca-sdk/examples/replace.rs), the +twin of tests/sensors/replace.mojo, is checked by the conformance suite as +tests/conformance/replace-rust.json, and Mojo's `morca.tables` reads its +table as one copy of each month. + The binary then also answers `sense` (a sensor's datasets) and `keys` (a keyed pipeline's keys). [rust/morca-sdk/examples/clean.rs](../rust/morca-sdk/examples/clean.rs) is a sensor, the twin of tests/sensors/clean.mojo; the conformance suite runs @@ -204,8 +239,6 @@ empty. later. - **dista tasks**: a dista task is Mojo code, and forking a threaded Rust process is the #135 hazard. A step parallelises inside itself instead. -- **`replaces=`**: iceberg-rust 0.11.0 has no overwrite or delete - transaction, so a step only appends. - **Object storage for logs**: `MORCA_LOGS`, `keep-log` and `put-log` take local paths; an `s3://` `MORCA_LOGS` keeps no attempt logs yet. - **Postgres over TLS**: the client connects without TLS diff --git a/pixi.toml b/pixi.toml index ed83506a..c124c602 100644 --- a/pixi.toml +++ b/pixi.toml @@ -487,8 +487,9 @@ test-rust-worker = { cmd = "tests/rust_worker.sh", depends-on = ["build-liblake" test-hindcast = { cmd = "tests/hindcast.sh", depends-on = ["build-liblake"] } # morca-sdk (#521): whole pipelines in Rust, with no Mojo toolchain; the # conformance suite on the Rust toy and a Rust sensor with tables, then the -# Mojo toy and the Rust toy side by side under one scheduler. -test-rust-sdk = { cmd = "scripts/with-postgres.sh tests/rust_sdk.sh" } +# Mojo toy and the Rust toy side by side under one scheduler. liblake is for +# the Mojo reader of the table a Rust step replaces. +test-rust-sdk = { cmd = "scripts/with-postgres.sh tests/rust_sdk.sh", depends-on = ["build-liblake"] } # MinIO for the tests over object storage (scripts/with-minio.sh). conda-forge # builds it for these two platforms only. diff --git a/rust/morca-sdk/Cargo.toml b/rust/morca-sdk/Cargo.toml index b15e3a68..fe03660d 100644 --- a/rust/morca-sdk/Cargo.toml +++ b/rust/morca-sdk/Cargo.toml @@ -62,5 +62,9 @@ required-features = ["tables"] name = "source" required-features = ["tables"] +[[example]] +name = "replace" +required-features = ["tables"] + [profile.release] strip = true diff --git a/rust/morca-sdk/README.md b/rust/morca-sdk/README.md index c1493b8b..38505c0d 100644 --- a/rust/morca-sdk/README.md +++ b/rust/morca-sdk/README.md @@ -37,8 +37,10 @@ cargo build --release && morca register target/release/days Postgres or a SQLite file, and passes the conformance suite (tests/conformance.py). - The `tables` feature adds Iceberg reads (pinned on a retry, recorded) and - appends (fenced, with the `orca.*` summary keys, found again on a retry) - through iceberg-rust 0.11.0 and arrow-rs 58, plus `sense` and `keys`. + commits (fenced, with the `orca.*` summary keys, found again on a retry) + through iceberg-rust 0.11.0 and arrow-rs 58, plus `sense` and `keys`. A + commit appends, or with `step(..).replaces([..])` replaces the partitions + it writes in one `overwrite` snapshot, as Mojo's `replaces=` does. - Rust 1.95 or later. docs/rust-pipelines.md in the morca repository is the guide, including what diff --git a/rust/morca-sdk/examples/replace.rs b/rust/morca-sdk/examples/replace.rs new file mode 100644 index 00000000..894f37ec --- /dev/null +++ b/rust/morca-sdk/examples/replace.rs @@ -0,0 +1,117 @@ +//! A sensor that replaces its partitions, the Rust twin of +//! tests/sensors/replace.mojo: reads raw.events and writes as many rows to +//! lake.monthly, partitioned by month, a month at a time, declaring +//! lake.monthly in `replaces` so a new run of a month leaves one copy of +//! its rows. The conformance suite runs it as +//! tests/conformance/replace-rust.json. +//! +//! It writes its files and commits them in two calls (`write_files`, then +//! `append_files`), so that with MORCA_REPLACE_DIE set it can die between +//! the two; MORCA_REPLACE_SLEEP makes each attempt take that many seconds +//! first. +//! +//! cargo build --release --features tables --example replace +//! replace rows each month in lake.monthly and its rows + +use std::collections::BTreeMap; +use std::sync::Arc; + +use morca_sdk::tables::arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray}; +use morca_sdk::tables::iceberg::arrow::schema_to_arrow_schema; +use morca_sdk::tables::iceberg::spec::{ + NestedField, PrimitiveType, Schema, Transform, Type, UnboundPartitionSpec, +}; +use morca_sdk::tables::read_as_other; +use morca_sdk::{Ctx, Month, Pipeline, StepResult, step}; + +const TABLE: &str = "lake.monthly"; + +fn schema() -> Schema { + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "month", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .expect("a valid schema") +} + +fn spec() -> UnboundPartitionSpec { + UnboundPartitionSpec::builder() + .with_spec_id(0) + .add_partition_field(2, "month", Transform::Identity) + .expect("a valid field") + .build() +} + +fn rows(month: &str, n: usize) -> Result { + let schema = Arc::new(schema_to_arrow_schema(&schema()).map_err(|e| e.to_string())?); + let ids: ArrayRef = Arc::new(Int64Array::from_iter_values(0..n as i64)); + let months: ArrayRef = Arc::new(StringArray::from_iter_values((0..n).map(|_| month))); + RecordBatch::try_new(schema, vec![ids, months]).map_err(|e| e.to_string()) +} + +fn load(ctx: &mut Ctx, m: Month) -> StepResult { + if let Some(s) = std::env::var("MORCA_REPLACE_SLEEP") + .ok() + .and_then(|s| s.parse::().ok()) + { + std::thread::sleep(std::time::Duration::from_secs_f64(s)); + } + let n: usize = ctx.read("raw.events")?.iter().map(|b| b.num_rows()).sum(); + ctx.ensure_table(TABLE, schema(), Some(spec()))?; + let files = if ctx.find_commit(TABLE)?.is_some() { + vec![] + } else { + ctx.write_files(TABLE, vec![rows(&m.to_string(), n)?])? + }; + if !files.is_empty() && std::env::var_os("MORCA_REPLACE_DIE").is_some() { + eprintln!("replace: dying between writing {m}'s files and committing them"); + // As a kill would: no attempt finished, nothing committed. + std::process::exit(70); + } + let snap = ctx.append_files(TABLE, files)?; + ctx.log(format!("{n} rows for {m} in snapshot {snap}")); + Ok(n as i64) +} + +/// `rows `: each month in lake.monthly and how many rows it has. +fn print_rows(wh: &str) -> Result<(), String> { + // SAFETY: single-threaded, before anything reads the environment. + unsafe { std::env::set_var("MORCA_WAREHOUSE", wh) }; + let mut by = BTreeMap::::new(); + for b in read_as_other(TABLE)? { + let c = b + .column_by_name("month") + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or("no month column")?; + for m in c.iter().flatten() { + *by.entry(m.to_string()).or_default() += 1; + } + } + for (m, n) in by { + println!("{m} {n}"); + } + Ok(()) +} + +fn main() { + let args: Vec = std::env::args().collect(); + if args.get(1).map(String::as_str) == Some("rows") { + let Some(wh) = args.get(2) else { + eprintln!("usage: replace rows "); + std::process::exit(2); + }; + if let Err(e) = print_rows(wh) { + eprintln!("replace: {e}"); + std::process::exit(1); + } + return; + } + let p = Pipeline::::new("replace_rust") + .start("2024-01") + .sensor() + .mapped(step("load").reads(["raw.events"]).replaces([TABLE]), load); + morca_sdk::main(p.build()); +} diff --git a/rust/morca-sdk/src/ctx.rs b/rust/morca-sdk/src/ctx.rs index b03322bd..9f75ac00 100644 --- a/rust/morca-sdk/src/ctx.rs +++ b/rust/morca-sdk/src/ctx.rs @@ -30,7 +30,10 @@ pub struct Ctx<'a> { /// The location of each commit this attempt made, in order. pub(crate) commits: Vec, pub(crate) declared_reads: Vec, + /// What the step writes, `replaces` among them. pub(crate) declared_writes: Vec, + /// The datasets whose partitions a new run of the step replaces. + pub(crate) declared_replaces: Vec, pub(crate) log_path: Option, pub(crate) store: &'a mut Owned, } @@ -151,6 +154,20 @@ impl Ctx<'_> { self.commits.iter().filter(|t| *t == location).count() } + /// Whether the step declared that it replaces `dataset`'s partitions + /// (`replaces`). + pub fn replaces(&self, dataset: &str) -> bool { + self.declared_replaces.iter().any(|d| d == dataset) + } + + /// `Ctx.same_run`: whether a snapshot with `summary` was committed by + /// any step of this run (the same pipeline and run). + pub fn same_run(&self, summary: &HashMap) -> bool { + self.run > 0 + && summary.get("orca.pipeline") == Some(&self.pipeline) + && summary.get("orca.run") == Some(&self.run.to_string()) + } + /// `Ctx.made`: whether a snapshot with `summary` is this step's next /// commit to the table at `location`, by any attempt of this run. Every /// key must be there; a run token, when both have one, must agree. diff --git a/rust/morca-sdk/src/exec.rs b/rust/morca-sdk/src/exec.rs index 48c46637..6051f93a 100644 --- a/rust/morca-sdk/src/exec.rs +++ b/rust/morca-sdk/src/exec.rs @@ -242,7 +242,8 @@ fn run_step( reads: vec![], commits: vec![], declared_reads: st.spec.reads.clone(), - declared_writes: st.spec.writes.clone(), + declared_writes: st.spec.all_writes(), + declared_replaces: st.spec.replaces.clone(), log_path: log, store, }; diff --git a/rust/morca-sdk/src/pipeline.rs b/rust/morca-sdk/src/pipeline.rs index 4249c995..2a77a84a 100644 --- a/rust/morca-sdk/src/pipeline.rs +++ b/rust/morca-sdk/src/pipeline.rs @@ -76,6 +76,7 @@ pub struct StepSpec { pub(crate) version: u64, pub(crate) reads: Vec, pub(crate) writes: Vec, + pub(crate) replaces: Vec, } /// A step named `name`, to set more on. @@ -125,6 +126,27 @@ impl StepSpec { self.writes = ds.into_iter().map(Into::into).collect(); self } + /// The warehouse datasets (`ns.table`) the step writes and whose + /// partitions a new run replaces rather than appends to, as Mojo's + /// `replaces=`: [`Ctx::append`] to one makes the run's first commit to + /// each partition an `overwrite` that removes what earlier runs left + /// there. They count as written, and needn't be in `writes` too. + pub fn replaces, S: Into>(mut self, ds: I) -> Self { + self.replaces = ds.into_iter().map(Into::into).collect(); + self + } + + /// Every dataset the step writes: `writes`, then each of `replaces` not + /// among them (`dataset_writes()` in src/morca/step.mojo). + pub(crate) fn all_writes(&self) -> Vec { + let mut out = self.writes.clone(); + for d in &self.replaces { + if !out.contains(d) { + out.push(d.clone()); + } + } + out + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -528,15 +550,16 @@ impl Built { return bad(format!("two steps are named {n}")); } names.push(n); - for d in s.spec.reads.iter().chain(&s.spec.writes) { + let writes = s.spec.all_writes(); + for d in s.spec.reads.iter().chain(&writes) { if !is_dataset(d) { return bad(format!("step {n}: '{d}' isn't a dataset (namespace.table)")); } } - if s.spec.writes.iter().any(|d| s.spec.reads.contains(d)) { + if writes.iter().any(|d| s.spec.reads.contains(d)) { return bad(format!("step {n} reads a dataset it writes")); } - if !cfg!(feature = "tables") && !(s.spec.reads.is_empty() && s.spec.writes.is_empty()) { + if !cfg!(feature = "tables") && !(s.spec.reads.is_empty() && writes.is_empty()) { return bad(format!( "step {n} declares datasets: build morca-sdk with the `tables` feature" )); @@ -587,11 +610,15 @@ impl Built { /// Each dataset a step reads and no step writes, once, in step order, /// then a keyed pipeline's keys dataset. pub(crate) fn sensed(&self) -> Vec { - let written: Vec<&String> = self.steps.iter().flat_map(|s| &s.spec.writes).collect(); + let written: Vec = self + .steps + .iter() + .flat_map(|s| s.spec.all_writes()) + .collect(); let mut out: Vec = vec![]; for s in &self.steps { for d in &s.spec.reads { - if !written.contains(&d) && !out.contains(d) { + if !written.contains(d) && !out.contains(d) { out.push(d.clone()); } } @@ -621,7 +648,10 @@ impl Built { for d in &s.spec.reads { all.push_str(&format!("\treads {d}")); } - for d in &s.spec.writes { + // A dataset in `replaces` is hashed as a write, as the Mojo + // binary hashes it (#295): replacing changes what a rerun does, + // not what the runs made. + for d in s.spec.all_writes() { all.push_str(&format!("\twrites {d}")); } if s.spec.version > 0 { @@ -686,7 +716,11 @@ impl Built { o.push(("version", J::v(s.spec.version))); } o.push(("reads", J::v(&s.spec.reads))); - o.push(("writes", J::v(&s.spec.writes))); + o.push(("writes", J::v(s.spec.all_writes()))); + // Only when declared, as the Mojo binary writes it. + if !s.spec.replaces.is_empty() { + o.push(("replaces", J::v(&s.spec.replaces))); + } J::Obj(o) }) .collect(); @@ -817,6 +851,26 @@ mod tests { assert!(empty.is_err()); } + // A step may declare datasets only with the `tables` feature. + #[cfg(feature = "tables")] + #[test] + fn replaces_is_a_write() { + let with = |spec: StepSpec| Pipeline::::new("t").mapped(spec, days).build(); + let writes = with(step("a").writes(["lake.x"])).unwrap(); + let replaces = with(step("a").replaces(["lake.x"])).unwrap(); + let both = with(step("a").writes(["lake.x"]).replaces(["lake.x"])).unwrap(); + // Hashed as a write, as the Mojo binary hashes it (#295). + assert_eq!(writes.registry_hash(), replaces.registry_hash()); + assert_eq!(writes.registry_hash(), both.registry_hash()); + let d: Value = serde_json::from_str(&replaces.describe()).unwrap(); + assert_eq!(d["steps"][0]["writes"], json!(["lake.x"])); + assert_eq!(d["steps"][0]["replaces"], json!(["lake.x"])); + let d: Value = serde_json::from_str(&writes.describe()).unwrap(); + assert!(d["steps"][0].get("replaces").is_none()); + let bad = with(step("a").reads(["lake.x"]).replaces(["lake.x"])); + assert!(bad.err().unwrap().contains("reads a dataset it writes")); + } + #[test] fn edges_checked_at_run_time() { let out = encode(&7u32).unwrap(); diff --git a/rust/morca-sdk/src/tables/catalog.rs b/rust/morca-sdk/src/tables/catalog.rs index f871a69d..d52bd3c4 100644 --- a/rust/morca-sdk/src/tables/catalog.rs +++ b/rust/morca-sdk/src/tables/catalog.rs @@ -30,6 +30,7 @@ //! Data and manifest files go through iceberg-rust's own `FileIO`. use std::collections::HashMap; +use std::str::FromStr; use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -39,8 +40,8 @@ use iceberg::io::FileIO; use iceberg::spec::{TableMetadata, TableMetadataBuilder}; use iceberg::table::Table; use iceberg::{ - Catalog, Error, ErrorKind, Namespace, NamespaceIdent, Result, Runtime, TableCommit, - TableCreation, TableIdent, + Catalog, Error, ErrorKind, MetadataLocation, Namespace, NamespaceIdent, Result, Runtime, + TableCommit, TableCreation, TableIdent, }; use object_store::aws::AmazonS3; use object_store::list::{PaginatedListOptions, PaginatedListStore}; @@ -338,6 +339,50 @@ impl MorcaCatalog { Ok(true) } + /// Commits `metadata`, built on `base`, as the table's next version + /// (`-.metadata.json`), as `update_table` commits a + /// transaction: for a snapshot iceberg-rust has no transaction for + /// (`overwrite`). Refuses, retryably, unless `base` is still the + /// table's current metadata. + pub async fn commit_next(&self, base: &Table, metadata: TableMetadata) -> Result { + let current = self.load_table(base.identifier()).await?; + let at = base.metadata_location_result()?; + if current.metadata_location() != Some(at) { + return Err(conflict(format!( + "{} moved on from {at}; reload and retry", + base.identifier() + ))); + } + let path = MetadataLocation::from_str(at)? + .with_next_version() + .try_with_new_metadata(&metadata)? + .to_string(); + let staged = self.table(base.identifier().clone(), path, metadata)?; + self.write_staged(staged).await + } + + /// Writes a staged table's metadata as the version its location names, + /// unless another writer committed that version first. + async fn write_staged(&self, staged: Table) -> Result
{ + let path = staged.metadata_location_result()?.to_string(); + let (dir, name) = path.rsplit_once('/').unwrap(); + let v = version_of(name).unwrap(); + if !self.files_of_version(dir, v).await?.is_empty() + || self.read_hint(dir).await.is_some_and(|h| h >= v) + { + return Err(conflict(format!( + "another writer committed version {v} first" + ))); + } + let doc = Bytes::from(serde_json::to_vec(staged.metadata())?); + if !self.claim_and_write(dir, v, &path, doc).await? { + return Err(conflict(format!( + "several writers claimed version {v}; reload and retry" + ))); + } + Ok(staged) + } + fn table(&self, ident: TableIdent, location: String, metadata: TableMetadata) -> Result
{ Table::builder() .file_io(self.file_io.clone()) @@ -466,22 +511,6 @@ impl Catalog for MorcaCatalog { e } })?; - let path = staged.metadata_location_result()?.to_string(); - let (dir, name) = path.rsplit_once('/').unwrap(); - let v = version_of(name).unwrap(); - if !self.files_of_version(dir, v).await?.is_empty() - || self.read_hint(dir).await.is_some_and(|h| h >= v) - { - return Err(conflict(format!( - "another writer committed version {v} first" - ))); - } - let doc = Bytes::from(serde_json::to_vec(staged.metadata())?); - if !self.claim_and_write(dir, v, &path, doc).await? { - return Err(conflict(format!( - "several writers claimed version {v}; reload and retry" - ))); - } - Ok(staged) + self.write_staged(staged).await } } diff --git a/rust/morca-sdk/src/tables/imp.rs b/rust/morca-sdk/src/tables/imp.rs index 0d5db1d0..6a32e77c 100644 --- a/rust/morca-sdk/src/tables/imp.rs +++ b/rust/morca-sdk/src/tables/imp.rs @@ -32,6 +32,7 @@ use object_store::local::LocalFileSystem; use parquet::file::properties::WriterProperties; use super::catalog::MorcaCatalog; +use super::replace; use crate::ctx::Ctx; use crate::pipeline::{Built, StepResult, fail}; @@ -275,44 +276,183 @@ impl Ctx<'_> { /// committed with the `orca.*` summary and recorded. If an earlier /// attempt of this run already made this commit, records that one and /// writes nothing. + /// + /// To a dataset the step declared in `replaces`, it's + /// [`Ctx::replace_partitions`]: the partitions the rows land in lose + /// what earlier runs left there. pub fn append(&mut self, dataset: &str, batches: Vec) -> StepResult { + let replace = self.replaces(dataset); + self.commit_rows(dataset, batches, replace) + } + + /// Writes `batches` to the dataset as one snapshot that also removes the + /// files earlier runs left in the partitions the rows land in, and + /// returns it, or the snapshot an earlier attempt of this run made (Mojo's + /// `replace_partitions`). What [`Ctx::append`] does to a dataset in + /// `replaces`; this is for one the step declares in `writes`. + /// + /// The run's first commit to a partition is an `overwrite` that removes + /// every file in it this run didn't commit; later commits of the run to + /// it append, since files of this run are never removed. An + /// unpartitioned table is one partition. If another writer committed a + /// file to one of the partitions after this run's first commit to the + /// table, two writers are replacing it at once: the attempt fails, + /// committing nothing. The summary has `orca.replaced-files` and + /// `orca.replaced-records`. + pub fn replace_partitions( + &mut self, + dataset: &str, + batches: Vec, + ) -> StepResult { + self.commit_rows(dataset, batches, true) + } + + /// Writes `batches` into the dataset's data files as Parquet, split by + /// its partition spec, without committing them: for + /// [`Ctx::append_files`] to commit. Call [`Ctx::find_commit`] first, so + /// a retry of an attempt that committed doesn't write them again. + pub fn write_files( + &mut self, + dataset: &str, + batches: Vec, + ) -> StepResult> { self.declared(dataset, true)?; - let cat = catalog()?; - let id = ident(dataset)?; - let mut t = block(load(&cat, &id))?.ok_or_else(|| { + let t = self.writable(&catalog()?, dataset)?; + block(write_files(&t, &batches)) + .map_err(|e| fail(format!("morca: can't write {dataset}'s files: {e}"))) + } + + /// Commits data files already written ([`Ctx::write_files`]) to the + /// dataset as one snapshot, as [`Ctx::append`] commits what it writes: + /// fenced, with the `orca.*` summary, recorded, or the commit an earlier + /// attempt made. To a dataset in `replaces`, it replaces their + /// partitions. + pub fn append_files(&mut self, dataset: &str, files: Vec) -> StepResult { + let replace = self.replaces(dataset); + self.commit_data_files(dataset, files, replace) + } + + /// [`Ctx::replace_partitions`] of data files already written. + pub fn replace_partition_files( + &mut self, + dataset: &str, + files: Vec, + ) -> StepResult { + self.commit_data_files(dataset, files, true) + } + + /// The table of a dataset the step writes, which must exist. + fn writable(&self, cat: &MorcaCatalog, dataset: &str) -> StepResult
{ + block(load(cat, &ident(dataset)?))?.ok_or_else(|| { fail(format!( "morca: there's no table {dataset}: ensure_table it first" )) - })?; - let location = t.metadata().location().to_string(); - if let Some(found) = self.found(t.metadata()) { - self.log_to_file(&format!( - "found commit {found} to {dataset} from an earlier attempt" - )); - self.record_commit(&location, found, dataset)?; + }) + } + + /// The commit an earlier attempt of the run made, recorded again. + fn found_again(&mut self, t: &Table, dataset: &str) -> StepResult> { + let Some(found) = self.found(t.metadata()) else { + return Ok(None); + }; + self.log_to_file(&format!( + "found commit {found} to {dataset} from an earlier attempt" + )); + self.record_commit(t.metadata().location(), found, dataset)?; + Ok(Some(found)) + } + + fn commit_rows( + &mut self, + dataset: &str, + batches: Vec, + replace: bool, + ) -> StepResult { + self.declared(dataset, true)?; + let cat = catalog()?; + let t = self.writable(&cat, dataset)?; + if let Some(found) = self.found_again(&t, dataset)? { return Ok(found); } let files = block(write_files(&t, &batches)) .map_err(|e| fail(format!("morca: can't write {dataset}'s files: {e}")))?; + self.commit(&cat, t, dataset, files, replace) + } + + fn commit_data_files( + &mut self, + dataset: &str, + files: Vec, + replace: bool, + ) -> StepResult { + self.declared(dataset, true)?; + let cat = catalog()?; + let t = self.writable(&cat, dataset)?; + if let Some(found) = self.found_again(&t, dataset)? { + return Ok(found); + } + self.commit(&cat, t, dataset, files, replace) + } + + /// Commits `files` to `t` as one snapshot, an append or (`replace`) one + /// replacing their partitions, reloading and trying again (planned + /// again) when another writer commits first, and records it. + fn commit( + &mut self, + cat: &MorcaCatalog, + mut t: Table, + dataset: &str, + files: Vec, + replace: bool, + ) -> StepResult { + let id = ident(dataset)?; + let location = t.metadata().location().to_string(); let mut tries = 0; + let mut replaced = (0, 0); let snap = loop { + let mut summary = self.summary(&location); + // Planned against the snapshot this try builds on, never one an + // earlier try saw: a stale list would leave another writer's + // rows beside these. + let plan = if replace { + let p = block(replace::plan(&t, &files, |s| self.same_run(s), dataset)) + .map_err(fail)?; + summary.insert("orca.replaced-files".into(), p.files.to_string()); + summary.insert("orca.replaced-records".into(), p.records.to_string()); + replaced = (p.files, p.records); + Some(p).filter(|p| p.removed > 0) + } else { + None + }; self.fence()?; - let summary = self.summary(&location); - let tx = Transaction::new(&t); - let tx = tx - .fast_append() - .add_data_files(files.clone()) - .set_snapshot_properties(summary) - .apply(tx)?; - match block(tx.commit(&cat)) { + let got = match &plan { + Some(p) => block(replace::commit_overwrite( + cat, + &t, + files.clone(), + p, + summary, + )), + None => { + let tx = Transaction::new(&t); + let tx = tx + .fast_append() + .add_data_files(files.clone()) + .set_snapshot_properties(summary) + .apply(tx)?; + block(tx.commit(cat)) + } + }; + match got { Ok(t2) => break current(t2.metadata()), Err(e) if e.retryable() && tries < COMMIT_RETRIES => { tries += 1; let base = 0.01 * f64::from(1u32 << (tries - 1).min(6)); let wait = base * rand::random_range(0.5..1.5); std::thread::sleep(std::time::Duration::from_secs_f64(wait)); - t = block(load(&cat, &id))? + t = block(load(cat, &id))? .ok_or_else(|| fail(format!("morca: {dataset} went away")))?; + // A commit that landed after all is done, not tried again. if let Some(found) = self.found(t.metadata()) { break found; } @@ -326,6 +466,13 @@ impl Ctx<'_> { } }; self.record_commit(&location, snap, dataset)?; + if replaced.0 > 0 { + self.log(format!( + "replaced {} files ({} rows) of {dataset} that earlier runs committed, in \ + snapshot {snap}", + replaced.0, replaced.1 + )); + } Ok(snap) } } @@ -506,6 +653,31 @@ pub fn append_as_other( }) } +/// For tests and tools: the dataset's rows at its current snapshot, read as +/// a program that isn't a morca pipeline would (nothing pinned or +/// recorded). No table is no rows. +pub fn read_as_other(dataset: &str) -> Result, String> { + let cat = catalog()?; + let Some(t) = block(load(&cat, &ident(dataset)?))? else { + return Ok(vec![]); + }; + if t.metadata().current_snapshot_id().is_none() { + return Ok(vec![]); + } + block(async { + t.scan() + .select_all() + .build() + .map_err(|e| e.to_string())? + .to_arrow() + .await + .map_err(|e| e.to_string())? + .try_collect() + .await + .map_err(|e| e.to_string()) + }) +} + /// A two-column schema (`id` long, `name` string), as the examples use. pub fn id_name_schema() -> Schema { use iceberg::spec::{NestedField, PrimitiveType, Type}; diff --git a/rust/morca-sdk/src/tables/mod.rs b/rust/morca-sdk/src/tables/mod.rs index 5b8c2197..3de467a0 100644 --- a/rust/morca-sdk/src/tables/mod.rs +++ b/rust/morca-sdk/src/tables/mod.rs @@ -11,8 +11,10 @@ //! in `orca.commits`; before writing anything, `find_commit` looks for the //! commit an earlier attempt of the run already made, and records that //! instead. -//! - Only appends: iceberg-rust 0.11.0 has no overwrite or delete -//! transaction, so a step can't declare `replaces=` yet. +//! - To a dataset the step declares in `replaces`, the run's first commit +//! to each partition is an `overwrite` that removes what earlier runs left +//! there (`replace`): iceberg-rust 0.11.0 has no overwrite transaction, so +//! that snapshot is written from its parts. //! //! Without the feature a pipeline declares no datasets, `sense` prints //! nothing and `keys` can't be asked for. @@ -21,6 +23,8 @@ mod catalog; #[cfg(feature = "tables")] mod imp; +#[cfg(feature = "tables")] +mod replace; #[cfg(feature = "tables")] pub use imp::*; diff --git a/rust/morca-sdk/src/tables/replace.rs b/rust/morca-sdk/src/tables/replace.rs new file mode 100644 index 00000000..d3230a62 --- /dev/null +++ b/rust/morca-sdk/src/tables/replace.rs @@ -0,0 +1,388 @@ +//! Replacing partitions (`replaces`), as `replace_partitions` in +//! src/morca/_lake/impl.mojo does: iceberg-rust 0.11.0 has no overwrite +//! transaction, so the snapshot is written here from its lower-level parts +//! (manifests, a manifest list, the snapshot and its summary) and committed +//! through [`MorcaCatalog::commit_next`]. +//! +//! - The run's first commit to a partition removes every data and delete +//! file in it that this run didn't commit, and adds the new files, in one +//! `overwrite` snapshot. Later commits of the run to the partition find +//! only its own files there and append; files of this run, from any of its +//! steps and attempts, are never removed. +//! - The partitions are those of the new files under the table's current +//! spec: no rows replace nothing. Files written under an older spec are +//! left alone. An unpartitioned table is one partition. +//! - What to remove is planned against the snapshot each try commits on top +//! of, again on every retry after another writer committed first. +//! - A file in a partition being replaced that another writer committed +//! after this run's first commit to the table means two writers are +//! writing the partition at once: the commit fails, committing nothing. +//! +//! The summary carries `orca.replaced-files` and `orca.replaced-records`, +//! the data files and rows removed. + +use std::cmp::Ordering; +use std::collections::{HashMap, HashSet}; + +use iceberg::spec::{ + DataContentType, DataFile, DataFileFormat, Datum, FormatVersion, MAIN_BRANCH, + ManifestContentType, ManifestEntryRef, ManifestFile, ManifestListWriter, ManifestWriter, + ManifestWriterBuilder, Operation, PartitionSpecRef, PrimitiveType, Snapshot, SnapshotReference, + SnapshotRetention, SnapshotSummaryCollector, Struct, Summary, TableProperties, +}; +use iceberg::table::Table; +use iceberg::{Error, ErrorKind}; + +use super::catalog::MorcaCatalog; + +/// What a replace removes from the snapshot it's planned on. +pub(crate) struct Plan { + /// Manifests carried into the new snapshot as they are. + keep: Vec, + /// Manifests that hold a file to remove: each live entry, and whether + /// it goes. + rewrite: Vec<(ManifestFile, Vec<(ManifestEntryRef, bool)>)>, + /// Files removed, data and delete. + pub removed: usize, + /// Data files and rows removed. + pub files: u64, + pub records: u64, +} + +/// The partitions files land in (their `partition` under the default spec). +fn partitions(added: &[DataFile]) -> HashSet { + added.iter().map(|f| f.partition().clone()).collect() +} + +/// Whether manifest `mf` may hold files of one of `keys`, by its partition +/// summaries; the others aren't read. Unsure is yes. +fn may_hold(mf: &ManifestFile, types: &[Option], keys: &HashSet) -> bool { + let Some(sums) = &mf.partitions else { + return true; + }; + if sums.len() != types.len() || sums.is_empty() { + return true; + } + keys.iter().any(|k| { + k.fields().len() != sums.len() + || k.fields().iter().zip(sums).zip(types).all(|((v, s), t)| { + let Some(v) = v else { + return s.contains_null; + }; + let (Some(t), Some(v)) = (t, v.as_primitive_literal()) else { + return true; + }; + let (Some(lo), Some(hi)) = (&s.lower_bound, &s.upper_bound) else { + return false; + }; + let (Ok(lo), Ok(hi)) = ( + Datum::try_from_bytes(lo, t.clone()), + Datum::try_from_bytes(hi, t.clone()), + ) else { + return true; + }; + !matches!(lo.literal().partial_cmp(&v), Some(Ordering::Greater)) + && !matches!(hi.literal().partial_cmp(&v), Some(Ordering::Less)) + }) + }) +} + +/// Who committed snapshot `id`. +fn by_whom(t: &Table, id: i64) -> String { + match t.metadata().snapshot_by_id(id) { + None => "a snapshot no longer in the table's history".into(), + Some(s) => { + let p = &s.summary().additional_properties; + match p.get("orca.run") { + Some(run) => format!( + "run {run} of {}, step {}", + p.get("orca.pipeline").map_or("?", String::as_str), + p.get("orca.step").map_or("?", String::as_str) + ), + None => format!("snapshot {id}, not morca's"), + } + } + } +} + +/// The files a replace adding `added` removes from `t`'s current snapshot; +/// an error if another writer is writing one of its partitions. +/// `same_run` says whether a snapshot's summary is this run's. +pub(crate) async fn plan( + t: &Table, + added: &[DataFile], + same_run: impl Fn(&HashMap) -> bool, + name: &str, +) -> Result { + let mut out = Plan { + keep: vec![], + rewrite: vec![], + removed: 0, + files: 0, + records: 0, + }; + let md = t.metadata(); + let Some(current) = md.current_snapshot() else { + return Ok(out); + }; + let e = |e: Error| format!("morca: can't plan the replace of {name}: {e}"); + let list = t.manifest_list_reader(current).load().await.map_err(e)?; + if added.is_empty() { + out.keep = list.consume_entries().into_iter().collect(); + return Ok(out); + } + let spec = md.default_partition_spec(); + let ptype = spec.partition_type(md.current_schema()).map_err(e)?; + let types: Vec> = ptype + .fields() + .iter() + .map(|f| f.field_type.as_primitive_type().cloned()) + .collect(); + let keys = partitions(added); + // This run's snapshots, and where its first one is in the history. + let mut snaps: Vec<_> = md.snapshots().collect(); + snaps.sort_by_key(|s| s.sequence_number()); + let mut index = HashMap::new(); + let mut mine = HashSet::new(); + let mut first_mine = None; + for (i, s) in snaps.iter().enumerate() { + index.insert(s.snapshot_id(), i); + if same_run(&s.summary().additional_properties) { + mine.insert(s.snapshot_id()); + first_mine.get_or_insert(i); + } + } + for mf in list.consume_entries() { + if mf.partition_spec_id != spec.spec_id() || !may_hold(&mf, &types, &keys) { + out.keep.push(mf); + continue; + } + let manifest = t.manifest_reader().read(&mf).await.map_err(e)?; + let mut entries = vec![]; + let mut any = false; + for en in manifest.entries() { + if !en.is_alive() { + continue; + } + let snap = en.snapshot_id().unwrap_or(mf.added_snapshot_id); + let goes = keys.contains(en.data_file().partition()) && !mine.contains(&snap); + if goes { + if let (Some(first), Some(&at)) = (first_mine, index.get(&snap)) + && at > first + { + return Err(format!( + "morca: another writer is writing a partition of {name} that this \ + run replaces: {} committed {} after this run's first commit to it. \ + Nothing was committed; rerun once the other has finished", + by_whom(t, snap), + en.file_path() + )); + } + any = true; + out.removed += 1; + if en.content_type() == DataContentType::Data { + out.files += 1; + out.records += en.record_count(); + } + } + entries.push((en.clone(), goes)); + } + if any { + out.rewrite.push((mf, entries)); + } else { + out.keep.push(mf); + } + } + Ok(out) +} + +fn new_snapshot_id(t: &Table) -> i64 { + loop { + let (a, b) = uuid::Uuid::new_v4().as_u64_pair(); + let id = ((a ^ b) >> 1) as i64; + if id > 0 && t.metadata().snapshot_by_id(id).is_none() { + return id; + } + } +} + +fn manifest_writer( + t: &Table, + path: String, + snap: i64, + spec: &PartitionSpecRef, + content: ManifestContentType, +) -> iceberg::Result { + let md = t.metadata(); + let b = ManifestWriterBuilder::new( + t.file_io().new_output(path)?, + Some(snap), + md.current_schema().clone(), + spec.as_ref().clone(), + ); + Ok(match (md.format_version(), content) { + (FormatVersion::V1, _) => b.build_v1(), + (FormatVersion::V2, ManifestContentType::Data) => b.build_v2_data(), + (FormatVersion::V2, ManifestContentType::Deletes) => b.build_v2_deletes(), + (FormatVersion::V3, ManifestContentType::Data) => b.build_v3_data(), + (FormatVersion::V3, ManifestContentType::Deletes) => b.build_v3_deletes(), + }) +} + +/// `total-*` from the parent's total and this snapshot's changes, as +/// iceberg's `update_totals`: none when the parent has no total. +fn totals(props: &mut HashMap, prev: Option<&Summary>) { + for (total, add, del) in [ + ("total-data-files", "added-data-files", "deleted-data-files"), + ( + "total-delete-files", + "added-delete-files", + "removed-delete-files", + ), + ("total-records", "added-records", "deleted-records"), + ("total-files-size", "added-files-size", "removed-files-size"), + ( + "total-position-deletes", + "added-position-deletes", + "removed-position-deletes", + ), + ( + "total-equality-deletes", + "added-equality-deletes", + "removed-equality-deletes", + ), + ] { + let base = match prev { + None => 0, + Some(p) => match p.additional_properties.get(total).map(|v| v.parse::()) { + Some(Ok(v)) => v, + _ => continue, + }, + }; + let n = |k: &str| { + props + .get(k) + .and_then(|v| v.parse::().ok()) + .unwrap_or(0) + }; + let v = (base + n(add)).saturating_sub(n(del)); + props.insert(total.to_string(), v.to_string()); + } +} + +/// Commits `added` to `t` as one `overwrite` snapshot that removes what +/// `plan` found, with `summary` (the `orca.*` keys) beside iceberg's own +/// counts, and returns the table as committed. +pub(crate) async fn commit_overwrite( + cat: &MorcaCatalog, + t: &Table, + added: Vec, + plan: &Plan, + summary: HashMap, +) -> iceberg::Result
{ + let md = t.metadata(); + if t.encryption_manager().is_some() { + return Err(Error::new( + ErrorKind::FeatureUnsupported, + "morca: replacing partitions of an encrypted table", + )); + } + let snap = new_snapshot_id(t); + let commit = uuid::Uuid::new_v4(); + let dir = md.metadata_location()?; + let schema = md.current_schema().clone(); + let spec = md.default_partition_spec().clone(); + let mut collector = SnapshotSummaryCollector::default(); + collector.set_partition_summary_limit( + md.properties() + .get(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT) + .and_then(|v| v.parse().ok()) + .unwrap_or(TableProperties::PROPERTY_WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT), + ); + let mut n = 0..; + let mut path = || { + format!( + "{dir}/{commit}-m{}.{}", + n.next().unwrap(), + DataFileFormat::Avro + ) + }; + let mut manifests = vec![]; + if !added.is_empty() { + let mut w = manifest_writer(t, path(), snap, &spec, ManifestContentType::Data)?; + for f in added { + collector.add_file(&f, schema.clone(), spec.clone()); + // Its sequence numbers are inherited from the manifest list. + w.add_file(f, -1)?; + } + manifests.push(w.write_manifest_file().await?); + } + for (mf, entries) in &plan.rewrite { + let mut w = manifest_writer(t, path(), snap, &spec, mf.content)?; + for (en, goes) in entries { + let seq = en.sequence_number().unwrap_or(mf.sequence_number); + let file_seq = en.file_sequence_number.or(Some(seq)); + let f = en.data_file().clone(); + if *goes { + collector.remove_file(&f, schema.clone(), spec.clone()); + w.add_delete_file(f, seq, file_seq)?; + } else { + let id = en.snapshot_id().unwrap_or(mf.added_snapshot_id); + w.add_existing_file(f, id, seq, file_seq)?; + } + } + manifests.push(w.write_manifest_file().await?); + } + manifests.extend(plan.keep.iter().cloned()); + + let list_path = format!("{dir}/snap-{snap}-0-{commit}.{}", DataFileFormat::Avro); + let parent = md.current_snapshot_id(); + let seq = md.next_sequence_number(); + let first_row = md.next_row_id(); + let writer = t.file_io().new_output(list_path.clone())?.writer().await?; + let mut list = match md.format_version() { + FormatVersion::V1 => ManifestListWriter::v1(writer, snap, parent), + FormatVersion::V2 => ManifestListWriter::v2(writer, snap, parent, seq), + FormatVersion::V3 => ManifestListWriter::v3(writer, snap, parent, seq, Some(first_row)), + }; + list.add_manifests(manifests.into_iter())?; + let next_row = list.next_row_id(); + list.close().await?; + + let mut props = summary; + props.extend(collector.build()); + totals(&mut props, md.current_snapshot().map(|s| s.summary())); + let b = Snapshot::builder() + .with_manifest_list(list_path) + .with_snapshot_id(snap) + .with_parent_snapshot_id(parent) + .with_sequence_number(seq) + .with_summary(Summary { + operation: Operation::Overwrite, + additional_properties: props, + }) + .with_schema_id(md.current_schema_id()) + .with_timestamp_ms(now_ms()); + let snapshot = match next_row { + Some(r) => b.with_row_range(first_row, r - first_row).build(), + None => b.build(), + }; + let metadata = md + .clone() + .into_builder(Some(t.metadata_location_result()?.to_string())) + .add_snapshot(snapshot)? + .set_ref( + MAIN_BRANCH, + SnapshotReference::new(snap, SnapshotRetention::branch(None, None, None)), + )? + .build()? + .metadata; + cat.commit_next(t, metadata).await +} + +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} diff --git a/tests/conformance.py b/tests/conformance.py index 21ffb1bc..087936a1 100755 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -13,7 +13,9 @@ "partitions": "monthly", "keys": ["2024-01", "2024-02"], the keys a run covers "steps": [{"name": "days", "kind": "mapped", - "reads": [], "writes": []}, ...], in run order + "reads": [], "writes": [], + "replaces": []}, ...], in run order; a dataset in + `replaces` counts as written "sensor": false, "env": {...}, set for every command "slow_env": {...}, makes the first step's attempts take 3 s or @@ -22,8 +24,12 @@ checks are skipped "warehouse": true, runs get a fresh MORCA_WAREHOUSE "setup": [[cmd...]], run once per warehouse, before anything - "commit_input": [cmd...]} commits a new snapshot to a table a step + "commit_input": [cmd...], commits a new snapshot to a table a step reads; "{warehouse}" in these is replaced + "die_env": {...}, makes the step that replaces die between + writing its files and committing them + "rows": [[cmd...]]} readers of the dataset a step replaces, each + printing ` ` lines Runs are started only as a runner starts them: the suite inserts the run row the way the scheduler does (`running`, owner `scheduler`, no @@ -384,6 +390,13 @@ def until(fn, seconds, what): # --- Expectations from the manifest --- +def writes_of(s): + """What a manifest's step writes: `writes`, then `replaces`.""" + out = list(s.get("writes", [])) + out += [d for d in s.get("replaces", []) if d not in out] + return out + + def expected_steps(m): """(step, partition, ord) of a whole run over the manifest's keys.""" out = set() @@ -398,7 +411,7 @@ def expected_steps(m): def sensed(m): """Each dataset a step reads and no step writes, once, in step order.""" - written = {d for s in m["steps"] for d in s.get("writes", [])} + written = {d for s in m["steps"] for d in writes_of(s)} out = [] for s in m["steps"]: for d in s.get("reads", []): @@ -1092,7 +1105,7 @@ def commits(): got = rows("commits", run) seen = {} for aid, s, p, loc, snap, ds in got: - need(ds in by_step[s].get("writes", []), + need(ds in writes_of(by_step[s]), f"{s} committed to {ds or loc}, which describe doesn't list") summ = snapshots(loc).get(int(snap), {}).get("summary") need(summ is not None, f"{s} [{p}]: snapshot {snap} isn't in {loc}") @@ -1107,7 +1120,7 @@ def commits(): need(not bad, f"{s} [{p}]'s commit {snap}: {bad}, not " f"{ {k: want[k] for k in bad} }") for s in m["steps"]: - for ds in s.get("writes", []): + for ds in writes_of(s): need(any(x[1] == s["name"] and x[5] == ds for x in got), f"{s['name']} recorded no commit to {ds}") @@ -1133,7 +1146,7 @@ def reads(): # A crash after a step committed: its attempt left running, then a # new snapshot of what it reads, then a resume. - writer = next((s for s in m["steps"] if s.get("writes")), None) + writer = next((s for s in m["steps"] if writes_of(s)), None) ctx = {} def crashed_after_commit(): @@ -1204,6 +1217,146 @@ def pinned(): "a later attempt reads the snapshots the first attempt read, not " "the newer ones", pinned, where=w) + self.replaces(db, rows) + + # Replacing partitions: a step with `replaces`. + + def replaces(self, db, rows): + b, m, w = self.b, self.b.m, db.where + keys = m["keys"] + step = next((s for s in m["steps"] if s.get("replaces")), None) + ids = ("tables.replace", "tables.replace_rerun", "tables.replace_die_before_commit", + "tables.replace_readers") + if step is None: + for cid in ids: + self.check(cid, "replacing-partitions", "", + lambda: _skip("no step replaces a table"), where=w) + return + locs = {ds: dataset_location(b.warehouse, ds) for ds in step["replaces"]} + + def resume(r, extra=None): + code, out, err = b.run(["resume", str(r)], db, extra=extra, timeout=300) + return code, (out + err).strip()[-300:] + + def current(loc): + md = metadata(loc) + snap = md and md.get("current-snapshot-id") + return snaps_of(loc).get(snap, {}).get("summary", {}) if snap else {} + + def snaps_of(loc): + return snapshots(loc) + + def commits_of(r): + """{dataset: {partition: summary}} of the run's commits to what + the step replaces, its last commit to each partition.""" + out = {ds: {} for ds in locs} + for _, s, p, loc, snap, ds in rows("commits", r): + if s == step["name"] and ds in locs: + out[ds][p] = snaps_of(loc)[int(snap)]["summary"] + return out + + def one_copy(r, what): + """The tables hold only what run `r` added: one copy of each key.""" + for ds, by in commits_of(r).items(): + need(set(by) == set(keys) if step["kind"] == "mapped" else by, + f"{what}: run {r} committed {ds} for {sorted(by)}, not {keys}") + added = sum(int(x.get("added-records", 0)) for x in by.values()) + total = int(current(locs[ds]).get("total-records", -1)) + need(total == added, f"{what}: {ds} has {total} rows, but run {r} " + f"added {added}: the runs before it weren't replaced") + + ctx = {} + + def replace(): + r = db.new_run(b.name(), b.hash(), keys) + code, out = resume(r) + need(db.run_row(r)["status"] == "succeeded", f"run {r} exited {code}: {out}") + for ds, by in commits_of(r).items(): + for p, summ in by.items(): + for k in ("orca.replaced-files", "orca.replaced-records"): + need(k in summ, f"{ds} [{p}]'s commit has no {k}") + n = int(summ["orca.replaced-files"]) + need(n > 0, f"{ds} [{p}]: replaced no files, though earlier runs " + "wrote the partition") + need(summ.get("operation") == "overwrite", + f"{ds} [{p}]'s commit is `{summ.get('operation')}`, not overwrite") + need(int(summ.get("deleted-data-files", 0)) == n, + f"{ds} [{p}]: deleted-data-files {summ.get('deleted-data-files')}, " + f"orca.replaced-files {n}") + need(int(summ.get("deleted-records", 0)) == + int(summ["orca.replaced-records"]), + f"{ds} [{p}]: deleted-records {summ.get('deleted-records')}, " + f"orca.replaced-records {summ['orca.replaced-records']}") + one_copy(r, "after a new run") + ctx["run"] = r + + self.check("tables.replace", "replacing-partitions", + "a new run's commit to a partition it replaces is an `overwrite` " + "carrying orca.replaced-files and orca.replaced-records, and leaves " + "one copy of the partition's rows", replace, where=w) + + def rerun(): + if "run" not in ctx: + raise Skip("no replacing run") + r = db.new_run(b.name(), b.hash(), keys) + code, out = resume(r) + need(db.run_row(r)["status"] == "succeeded", f"run {r} exited {code}: {out}") + one_copy(r, "after another run of the same keys") + ctx["run"] = r + + self.check("tables.replace_rerun", "replacing-partitions", + "running the same keys again still leaves one copy", rerun, where=w) + + def die_before_commit(): + if not m.get("die_env"): + raise Skip("no die_env in the manifest") + before = {ds: len(snaps_of(loc)) for ds, loc in locs.items()} + r = db.new_run(b.name(), b.hash(), keys) + code, out = resume(r, extra=m["die_env"]) + need(code != 0, f"the run didn't die (exit {code}): {out}") + for ds, loc in locs.items(): + need(len(snaps_of(loc)) == before[ds], + f"{ds} got a snapshot from the run that died before committing") + row = db.run_row(r) + need(row["status"] == "running", f"the dead run is {row['status']}") + time.sleep(STALE + 0.3) + code, out = resume(r) + need(db.run_row(r)["status"] == "succeeded", + f"the resumed run exited {code}: {out}") + att = [x for x in db.attempts(r) if x[1] == step["name"]] + need(any(x[4] == "failed" for x in att) and att[-1][4] == "succeeded", + f"{step['name']}'s attempts are {[(x[2], x[3], x[4]) for x in att]}") + one_copy(r, "after a crash between writing and committing") + ctx["run"] = r + + self.check("tables.replace_die_before_commit", "replacing-partitions", + "a step that dies between writing its files and committing them " + "commits nothing; the resumed run commits once and leaves one copy", + die_before_commit, where=w) + + def readers(): + if not m.get("rows"): + raise Skip("no readers in the manifest") + if "run" not in ctx: + raise Skip("no replacing run") + (ds, by), = list(commits_of(ctx["run"]).items())[:1] + want = {p: int(x.get("added-records", 0)) for p, x in by.items()} + for cmd in m["rows"]: + p = subprocess.run(b.subst(cmd), env=b.env(), capture_output=True, + text=True, timeout=300) + need(p.returncode == 0, f"{cmd[0]} exited {p.returncode}: {p.stderr[-300:]}") + got = {} + for line in p.stdout.split("\n"): + if line.strip(): + k, n = line.split() + got[k] = int(n) + need(got == want, f"{os.path.basename(cmd[0])} reads {ds} as {got}, " + f"not {want}") + + self.check("tables.replace_readers", "replacing-partitions", + "other readers of the replaced table see one copy of each partition", + readers, where=w) + def _skip(why): raise Skip(why) @@ -1239,7 +1392,7 @@ def describe_fields(d): after = s.get("after") need(after is None if not names else after in names, f"{s['name']}: after {after!r}") - for k in ("reads", "writes"): + for k in ("reads", "writes", "replaces"): ds = s.get(k, []) need(isinstance(ds, list) and all(isinstance(x, str) and "." in x for x in ds), f"{s['name']}: `{k}` isn't a list of `ns.table` names") @@ -1252,10 +1405,10 @@ def describe_manifest(d, m): need(kind == m.get("partitions", "monthly"), f"partitions are {kind}") need(bool(d.get("sensor", False)) == bool(m.get("sensor", False)), f"sensor is {d.get('sensor')}") - got = [(s["name"], s["kind"], sorted(s.get("reads", [])), sorted(s.get("writes", []))) - for s in d["steps"]] - want = [(s["name"], s["kind"], sorted(s.get("reads", [])), sorted(s.get("writes", []))) - for s in m["steps"]] + got = [(s["name"], s["kind"], sorted(s.get("reads", [])), sorted(s.get("writes", [])), + sorted(s.get("replaces", []))) for s in d["steps"]] + want = [(s["name"], s["kind"], sorted(s.get("reads", [])), sorted(writes_of(s)), + sorted(s.get("replaces", []))) for s in m["steps"]] need(got == want, f"steps {got}, not {want}") diff --git a/tests/conformance.sh b/tests/conformance.sh index 0bf56718..e826e9c7 100755 --- a/tests/conformance.sh +++ b/tests/conformance.sh @@ -6,8 +6,10 @@ # examples toy, daily and fanout (the multi-call test binary, # scripts/pipelines.sh), then tests/chaos.sh in the suite's # mode, over Postgres and SQLite -# tables the sensor tests/sensors/clean.mojo, which reads raw.events -# and commits to lake.clean, in the data environment +# tables the sensors tests/sensors/clean.mojo, which reads raw.events +# and commits to lake.clean, and tests/sensors/replace.mojo, +# which replaces its months of lake.monthly, in the data +# environment # # Each pipeline runs over SQLite and over the Postgres MORCA_DATABASE_URL # names (pixi run test-conformance starts a throwaway server). @@ -32,7 +34,7 @@ examples) ;; tables) pids= - for p in source clean; do + for p in source clean replace rows_by; do # shellcheck disable=SC2086 scripts/mojo-build.sh -I src -I tests/sensors -I tests/lineage $DEP_INC "tests/sensors/$p.mojo" -o "build/sn-$p" & pids="$pids $!" @@ -40,6 +42,7 @@ tables) scripts/build-shared.sh morca & for pid in $pids $!; do wait "$pid"; done python3 tests/conformance.py build/sn-clean tests/conformance/clean.json || fail=1 + python3 tests/conformance.py build/sn-replace tests/conformance/replace.json || fail=1 ;; *) echo "usage: tests/conformance.sh [examples|tables]" >&2 diff --git a/tests/conformance/replace-rust.json b/tests/conformance/replace-rust.json new file mode 100644 index 00000000..999672ea --- /dev/null +++ b/tests/conformance/replace-rust.json @@ -0,0 +1,18 @@ +{ + "pipeline": "replace_rust", + "partitions": "monthly", + "keys": ["2024-03", "2024-04"], + "steps": [ + {"name": "load", "kind": "mapped", "reads": ["raw.events"], "replaces": ["lake.monthly"]} + ], + "sensor": true, + "warehouse": true, + "setup": [["build/cargo/release/examples/source", "{warehouse}", "2024-03"]], + "slow_env": {"MORCA_REPLACE_SLEEP": "3"}, + "commit_input": ["build/cargo/release/examples/source", "{warehouse}"], + "die_env": {"MORCA_REPLACE_DIE": "1"}, + "rows": [ + ["build/cargo/release/examples/replace", "rows", "{warehouse}"], + ["build/sn-rows_by", "{warehouse}", "lake.monthly", "month"] + ] +} diff --git a/tests/conformance/replace.json b/tests/conformance/replace.json new file mode 100644 index 00000000..017e5704 --- /dev/null +++ b/tests/conformance/replace.json @@ -0,0 +1,15 @@ +{ + "pipeline": "replace", + "partitions": "monthly", + "keys": ["2024-03", "2024-04"], + "steps": [ + {"name": "load", "kind": "mapped", "reads": ["raw.events"], "replaces": ["lake.monthly"]} + ], + "sensor": true, + "warehouse": true, + "setup": [["build/sn-source", "{warehouse}", "2024-03"]], + "slow_env": {"MORCA_REPLACE_SLEEP": "3"}, + "commit_input": ["build/sn-source", "{warehouse}"], + "die_env": {"MORCA_REPLACE_DIE": "1"}, + "rows": [["build/sn-rows_by", "{warehouse}", "lake.monthly", "month"]] +} diff --git a/tests/rust_sdk.sh b/tests/rust_sdk.sh index 2ce51b82..1f8ea34e 100755 --- a/tests/rust_sdk.sh +++ b/tests/rust_sdk.sh @@ -6,9 +6,13 @@ # and the registry hash, the crash summary), with and without tables; # - examples/toy-rust, examples/toy.mojo rebuilt in Rust, links no Mojo # runtime; -# - the conformance suite (tests/conformance.py) passes for the Rust toy and +# - the conformance suite (tests/conformance.py) passes for the Rust toy, # for the Rust sensor with tables (rust/morca-sdk/examples/clean.rs, the -# twin of tests/sensors/clean.mojo), over SQLite and Postgres; +# twin of tests/sensors/clean.mojo), and for the Rust sensor that replaces +# its partitions (examples/replace.rs, the twin of +# tests/sensors/replace.mojo), whose table Mojo's morca.tables +# (tests/sensors/rows_by.mojo) and iceberg-rust then read as one copy of +# each month, over SQLite and Postgres; # - side by side: the Mojo toy and the Rust toy registered with one morca # scheduler (the Rust one placed on the control host), fired by the same # schedule, then a backfill each, every run succeeding; `morca status` @@ -57,6 +61,10 @@ mojo_morca=$! # shellcheck disable=SC2086 scripts/mojo-build.sh -I src $DEP_INC examples/toy.mojo -o build/rust-sdk/toy >> "$log" 2>&1 & mojo_toy=$! +# The Mojo reader of the table the Rust replace example writes. +# shellcheck disable=SC2086 +scripts/mojo-build.sh -I src $DEP_INC tests/sensors/rows_by.mojo -o build/sn-rows_by >> "$log" 2>&1 & +mojo_rows=$! cargo test --manifest-path rust/morca-sdk/Cargo.toml >> "$log" 2>&1 || fail "cargo test" cargo test --manifest-path rust/morca-sdk/Cargo.toml --features tables >> "$log" 2>&1 \ @@ -77,10 +85,13 @@ echo "ok the Rust toy builds with cargo alone ($(wc -c < "$rtoy" | tr -d ' ') wait "$mojo_morca" || fail "build morca" wait "$mojo_toy" || fail "build the Mojo toy" +wait "$mojo_rows" || fail "build the Mojo reader (tests/sensors/rows_by.mojo)" python3 tests/conformance.py "$rtoy" tests/conformance/toy-rust.json || fail "conformance: the Rust toy" python3 tests/conformance.py "$CARGO_TARGET_DIR/release/examples/clean" tests/conformance/clean-rust.json \ || fail "conformance: the Rust sensor with tables" +python3 tests/conformance.py "$CARGO_TARGET_DIR/release/examples/replace" tests/conformance/replace-rust.json \ + || fail "conformance: the Rust sensor that replaces its partitions" # Side by side under one scheduler, in a database of their own. db=morca_side_by_side diff --git a/tests/sensors/replace.mojo b/tests/sensors/replace.mojo new file mode 100644 index 00000000..61967c90 --- /dev/null +++ b/tests/sensors/replace.mojo @@ -0,0 +1,78 @@ +"""A sensor that replaces its partitions: reads raw.events and writes as +many rows to lake.monthly, partitioned by month, a month at a time, with +lake.monthly in `replaces=`, so a new run of a month leaves one copy of its +rows. The twin of rust/morca-sdk/examples/replace.rs; the conformance suite +runs it as tests/conformance/replace.json. + +It writes its files and commits them in two calls (`write_files`, then +`append_files`), so that with MORCA_REPLACE_DIE set it can exit between the +two; MORCA_REPLACE_SLEEP makes each attempt take that many seconds first. +""" + +from std.os import getenv +from std.sys import exit +from std.time import sleep + +from morca import Ctx, Mapped, Month, Monthly, Pipeline, Step, morca_main +from morca.tables import ( + DataFiles, + PartitionField, + PartitionSpec, + Schema, + append_files, + ensure_table, + find_commit, + parse_transform, + table, + write_files, +) +from rows import SCHEMA, count, rows + +comptime TABLE = "lake.monthly" + + +def load(mut ctx: Ctx, m: Month) raises -> Int64: + var nap = getenv("MORCA_REPLACE_SLEEP") + if nap: + sleep(Float64(nap)) + var n = count(ctx, table(ctx, "raw.events")) + var t = ensure_table( + ctx, + TABLE, + Schema.parse(SCHEMA), + PartitionSpec( + 0, + [ + PartitionField.single( + 2, 1000, "month", parse_transform("identity") + ) + ], + ), + ) + var files = DataFiles() + if find_commit(ctx, t) < 0: + files = write_files(t, rows(String(m), n)) + if getenv("MORCA_REPLACE_DIE"): + print("replace: exiting between writing and committing", m) + # As a kill would: no attempt finished, nothing committed. + exit(70) + _ = append_files(ctx, t, files) + return Int64(n) + + +comptime Load = Step[ + "load", + Month, + Int64, + load, + Mapped, + reads=["raw.events"], + replaces=[TABLE], +] +comptime P = Pipeline[ + "replace", Load, partitions=Monthly["2024-01"], sensor=True +] + + +def main() raises: + morca_main[P]() diff --git a/tests/sensors/rows_by.mojo b/tests/sensors/rows_by.mojo new file mode 100644 index 00000000..dfab5f87 --- /dev/null +++ b/tests/sensors/rows_by.mojo @@ -0,0 +1,32 @@ +"""Reads a dataset with morca.tables, outside any step, and prints each +value of a string column with how many rows have it, sorted: the +conformance suite's Mojo reader of a table another binary replaced +(tests/conformance/replace*.json, "rows"). + + rows_by +""" + +from std.os import setenv +from std.sys import argv + +from morca.tables import load_dataset + + +def main() raises: + var args = argv() + if len(args) != 4: + raise Error("usage: rows_by ") + _ = setenv("MORCA_WAREHOUSE", String(args[1])) + var column = String(args[3]) + var by = Dict[String, Int]() + for b in load_dataset(String(args[2])).scan().to_batches(): + for i in range(b.num_columns()): + if b.name(i) == column: + for v in b.column_str(i)[0]: + by[v] = by.get(v, 0) + 1 + var keys = List[String]() + for k in by.keys(): + keys.append(k) + sort(keys) + for k in keys: + print(k, by[k])