diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ca81a64..bd53c0c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -694,7 +694,8 @@ jobs: # Rust workers (docs/rust-workers.md): the morca-worker crate's tests and # examples/rust-worker, a worker and its proxy pipeline over a local # warehouse (tests/rust_worker.sh), then examples/hindcast over its - # committed GRIB2 fixtures, offline (tests/hindcast.sh). In the rust + # committed GRIB2 fixtures, offline (tests/hindcast.sh), then morca-sdk's + # whole pipelines in Rust (tests/rust_sdk.sh). In the rust # environment: the data environment's tins plus conda-forge's rust (cargo # fetches crates; nothing fetches NOAA data). Where `changes` says, as # the test legs are. No secrets, and the token can only read. @@ -774,6 +775,8 @@ jobs: run: pixi run -e rust test-rust-worker - name: Hindcast (examples/hindcast, on fixtures) run: pixi run -e rust test-hindcast + - name: Pipelines in Rust (morca-sdk, conformance on SQLite and Postgres, beside the Mojo toy) + run: pixi run -e rust test-rust-sdk # The tests that apply on macOS, on the Mac mini runner (m4-mini, an isolated # `ci` user; no Docker, no sudo), for the repo's own code only: a fork's diff --git a/README.md b/README.md index 866ad1ef..d8a40254 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ them comes with a note in the changelog. Everything under `morca._internal` | [docs/placement.md](docs/placement.md) | where a pipeline's runs run: `default`, `ec2`, `control` and `worker:`, and the control host's limits | | [docs/workers.md](docs/workers.md) | remote workers: `morca worker`, worker tokens, a worker on another machine, the run's state over the API | | [docs/pipeline-protocol.md](docs/pipeline-protocol.md) | the pipeline-binary protocol: the subcommands, describe's JSON, the environment, the run store's rules and the tables' summary keys a pipeline binary must keep to | +| [docs/rust-pipelines.md](docs/rust-pipelines.md) | whole pipelines in Rust with the `morca-sdk` crate, no Mojo: steps, partitions, tables, and what version 1 leaves out | | [docs/rust-workers.md](docs/rust-workers.md) | steps written in Rust: the `morca-worker` crate, `morca.rust_step`, the C ABI, the schema file both sides read, and Parquet files a worker writes itself | | [docs/watchdog.md](docs/watchdog.md) | the watchdog's rules and thresholds, what a trip pauses, `morca watchdog status`, `ack` and `snooze` | | [docs/alerts.md](docs/alerts.md) | alerts to your phone through ntfy: what is sent, `MORCA_NOTIFY_*`, `morca notify` | @@ -87,6 +88,7 @@ pixi run -e data test-keyed-backfills # 20,000 keys backfilled a hundred to a pixi run -e data test-keyed-sensor # a keyed sensor over the keys a daily pipeline writes pixi run -e data test-keyed-names # keys such as owner/repo, in logs and the UI pixi run -e rust test-rust-worker # a step in Rust (docs/rust-workers.md): examples/rust-worker +pixi run -e rust test-rust-sdk # a whole pipeline in Rust (docs/rust-pipelines.md): examples/toy-rust pixi run -e rust test-hindcast # examples/hindcast: GFS forecasts fetched by a Rust worker and scored in Mojo, on fixtures pixi run toy && ./build/toy run --partitions 2024-01..2024-03 pixi run fanout && ./build/fanout run --partitions 2024-01 # steps calling dista diff --git a/changes/521-morca-sdk.minor.md b/changes/521-morca-sdk.minor.md new file mode 100644 index 00000000..8b85c21a --- /dev/null +++ b/changes/521-morca-sdk.minor.md @@ -0,0 +1 @@ +- **Pipelines in Rust**: the `morca-sdk` crate builds a whole pipeline in Rust, with no Mojo, into a binary the scheduler registers and runs (docs/rust-pipelines.md). (#521) diff --git a/docs/rust-pipelines.md b/docs/rust-pipelines.md new file mode 100644 index 00000000..56622e5f --- /dev/null +++ b/docs/rust-pipelines.md @@ -0,0 +1,217 @@ +# Pipelines in Rust + +A whole pipeline can be written in Rust, with no Mojo compiler and no Mojo +runtime: its steps, partitions, schedule and tables, built with `cargo` into +a pipeline binary that the prebuilt `morca` scheduler registers and runs like +any other. The `morca-sdk` crate, in [rust/morca-sdk](../rust/morca-sdk), is +what such a binary links. It speaks the [pipeline-binary +protocol](pipeline-protocol.md), version 1, and passes the conformance suite +(`tests/conformance.py`) on SQLite and Postgres. + +This is different from a [Rust worker](rust-workers.md), which is one step +in Rust inside a Mojo pipeline. With `morca-sdk` there is no Mojo at all. + +[examples/toy-rust](../examples/toy-rust) is [examples/toy.mojo](../examples/toy.mojo) +rebuilt this way, and `pixi run -e rust test-rust-sdk` (tests/rust_sdk.sh) +runs it beside the Mojo toy under one scheduler. + +## Writing a pipeline + +```toml +# Cargo.toml +[dependencies] +morca-sdk = { git = "https://github.com/morca-dev/morca" } # or a path +serde = { version = "1", features = ["derive"] } +``` + +```rust +use morca_sdk::{Cron, Ctx, Month, Pipeline, StepResult, fail, step}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +struct Measured { month: Month, days: u32 } + +fn days(_ctx: &mut Ctx, m: Month) -> StepResult { + Ok(Measured { days: m.days(), month: m }) +} + +fn measure(ctx: &mut Ctx, m: Measured) -> StepResult { + if m.days == 0 { + return Err(fail("no days")); + } + ctx.log(format!("{} has {} days", m.month, m.days)); + Ok(Measured { days: m.days * 24, month: m.month }) +} + +fn total(_ctx: &mut Ctx, ms: Vec) -> StepResult { + Ok(ms.iter().map(|m| m.days as i64).sum()) +} + +fn main() { + let toy = Pipeline::::new("toy_rust") + .start("2024-01") + .schedule(Cron::new("0 3 2 * *")) + .max_runs(2) + .mapped("days", days) + .mapped(step("measure").retries(2).timeout_seconds(600), measure) + .gather("total", total); + morca_sdk::main(toy.build()); +} +``` + +```sh +cargo build --release +./target/release/toy_rust describe +morca register ./target/release/toy_rust # or --placement control +``` + +From there the scheduler fires it, backfills it, and the UI shows its runs, +steps and logs, as for a Mojo pipeline. + +### Steps and their types + +A step is a function (or closure) `Fn(&mut Ctx, In) -> StepResult`. + +- **`mapped`** runs once per partition key. The first step takes the key + (`Month`, `Week`, `Day`, `Hour`, or `Name` for a keyed pipeline); a + mapped step after a mapped step takes its output for the same key. +- **`gather`** runs once per run. First, it takes the run's keys + (`Vec`); after a mapped step, its outputs in key order (`Vec`); + after a gather, its output. A mapped step can't follow a gather. + +The builder is typed, so a step whose input isn't what the step before it +makes doesn't compile. What Rust can't check at compile time is checked by +`build()`, which `main` calls before any command, so a broken definition +can't be registered (it exits 2): unique step names, datasets spelled +`namespace.table`, a start key that parses, a sensor that reads something, +a keyed pipeline's keys dataset. + +Inputs and outputs are `serde` types. A step's output is kept in its step +row as JSON, `{"type": , "value": ...}`, which is how a +resume hands it to the next step. A resume under new code +(`--allow-new-code`, or a rerun) decodes a stored output only into the type +it was stored as, so a step whose types changed fails with what it got +rather than misreading it. + +`step("name")` sets a step's `retries`, `backoff_seconds` (doubling), +`timeout_seconds`, data `version`, and its `reads` and `writes`. A plain +`"name"` takes the defaults: no retries, no timeout. + +A step fails by returning an error: `?` on any error type, or +`Err(fail("why"))`. A panic fails the attempt too, with its message. It +defers with `Err(defer(seconds, "why"))`: the attempt ends `deferred`, the +run starts again then, and no retries are spent. + +### The pipeline + +`Pipeline::::new(name)` and then, in any order before `build()`: + +| Call | describe | Default | +|---|---|---| +| `.start("2024-01")` | `partitions.start` | none | +| `.schedule(Cron::new("0 3 2 * *").tz("Europe/Paris"))` | `schedule` (`.fills_current()`, `.no_catchup()`) | none | +| `.max_runs(n)` | `max_runs` | 1 | +| `.run_retries(n)` | `run_retries` | 3 | +| `.retry_seconds(first, max)` | `retry_seconds`, `max_retry_seconds` | 0 | +| `.sensor()` | `sensor` | false | +| `.keys_per_run(n)` | `keys_per_run` | 1 | +| `.keys_from("ns.table", "column")` | `partitions.dataset`, `.column` (keyed) | required for `Name` | +| `.source(morca_sdk::source_from_build_env!())` | `source`, from `MORCA_SOURCE`, `MORCA_COMMIT`, `MORCA_DIRTY` at build time | empty | + +The registry hash is FNV-1a over the pipeline's name, each step's name, +kind, upstream, function, datasets and version, and the partition kind, as +the Mojo binary hashes its own; the function is the step's Rust path +(`toy_rust::measure`). The same source builds the same hash; a new step, +edge, function or dataset changes it. + +### The context + +`Ctx` has the step's `pipeline()`, `run()`, `step()`, `partition()`, +`attempt()` and `code()`, and: + +- `log(msg)`: a line on stdout tagged `[step partition]`, and in the + attempt's log with a timestamp, with the run's secrets masked; +- `progress(done, total)` and `metric(name, value)`, as log lines; +- `secret(name)`: a variable `MORCA_SECRETS` names, or None; +- `stopping()`: true once the run is outdated (newer code is registered, or + its scheduler instance drains; Postgres only). Poll it between units of + work, from any thread. What to do is the step's choice. + +These are the calls morca-worker's `RunContext` has for a worker step. A +step can parallelise inside itself with threads, rayon or tokio. + +### Tables + +With the crate's `tables` feature, a step reads and commits Iceberg tables +in `MORCA_WAREHOUSE` (a directory, or `s3://bucket/prefix` with the `AWS_*` +variables), with iceberg-rust 0.11.0 on arrow-rs 58: + +```rust +fn clean(ctx: &mut Ctx, m: Month) -> StepResult { + let n: usize = ctx.read("raw.events")?.iter().map(|b| b.num_rows()).sum(); + ctx.ensure_table("lake.clean", schema(), None)?; + ctx.append("lake.clean", vec![rows(&m, n)?])?; + Ok(n as i64) +} + +Pipeline::::new("clean_rust") + .sensor() + .mapped(step("clean").reads(["raw.events"]).writes(["lake.clean"]), clean) +``` + +- `read(dataset)` scans the snapshot this attempt already read, else the + one the step's first attempt read (so a retry or a resume sees the same + data), else the current one, and records the read. +- `append(dataset, batches)` writes Parquet split by the table's partition + spec, checks the commit fence, and commits one snapshot with the + `orca.*` summary keys, recorded as the attempt's commit. If an earlier + attempt of the run already made that commit, it records that one and + writes nothing. `find_commit(dataset)` asks first, for a step that writes + files itself. `arrow_schema(dataset)` is the schema the batches need. +- `ensure_table(dataset, schema, spec)` creates a table that isn't there. +- A step may read and write only the datasets it declares. + +The binary then also answers `sense` (a sensor's datasets) and `keys` (a +keyed pipeline's keys). [rust/morca-sdk/examples/clean.rs](../rust/morca-sdk/examples/clean.rs) +is a sensor, the twin of tests/sensors/clean.mojo; the conformance suite runs +it as tests/conformance/clean-rust.json. + +The crate picks arrow-rs 58 because iceberg-rust 0.11.0 is built on it, and +the batches a step hands iceberg must be the same major version. +morca-worker is on arrow 60, which doesn't matter here: a worker is a cdylib +that morca's Mojo side loads, and a morca-sdk binary never loads one. + +## Running + +A morca-sdk binary is a pipeline binary like any other: `morca register` +runs its `describe`, and the scheduler starts each run as +` resume `. It needs only `MORCA_DATABASE_URL` +(`postgres://...` or `sqlite:`), and `MORCA_WAREHOUSE` for tables. + +Over Postgres it checks that the schema has every column it uses, and +leaves migrations to the scheduler. Over SQLite it makes the file's schema +itself, as the Mojo binary does, so either can use a file the other made. + +` sample`, which the runners start beside each run, runs the +`morca sample` on the path; without one the run's Machines card stays +empty. + +## What isn't in version 1 + +- **Placements**: `default` (the scheduler's runner) and `control` only. + `MORCA_STATE_URL` (a worker pool's runs, over the UI's API), `ec2` + placement, the warm pool's commands and the Kubernetes runner come + later. +- **dista tasks**: a dista task is Mojo code, and forking a threaded Rust + process is the #135 hazard. A step parallelises inside itself instead. +- **`replaces=`**: iceberg-rust 0.11.0 has no overwrite or delete + transaction, so a step only appends. +- **Object storage for logs**: `MORCA_LOGS`, `keep-log` and `put-log` take + local paths; an `s3://` `MORCA_LOGS` keeps no attempt logs yet. +- **Postgres over TLS**: the client connects without TLS + (`sslmode=disable`), as to a database on the same host or network. +- **`run`, `status` and `fires`**, which are for people, not the + scheduler. +- **Mojo outputs**: a pipeline moved from Mojo to Rust is new code (its + functions differ), and can't resume a run a Mojo build started, since the + step outputs are encoded differently. diff --git a/examples/toy-rust/Cargo.lock b/examples/toy-rust/Cargo.lock new file mode 100644 index 00000000..a72ed48d --- /dev/null +++ b/examples/toy-rust/Cargo.lock @@ -0,0 +1,926 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a0cf66fd1eb2fb7b822cb78370d82303750809bc6ba855f3f58e517d3238db" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ef9e3e8154a45c7a95cff4c1299702106ad94062d7bb85fdc5d3a5afd91d45" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "ctutils", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06e69070b7e12262b0cd90832f0a7396f142a2b330181255562decf9f0c6215" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "morca-sdk" +version = "0.1.0" +dependencies = [ + "postgres", + "rusqlite", + "serde", + "serde_json", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "postgres" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ad20e0aa0b24f5a394eab4f78c781d248982b22b25cecc7e3aa46a681605bd" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353b74f9810d5b085b6c42c9d86a5f0fdb795553235cdd48753b32e3809d18a7" +dependencies = [ + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f172c4812a19b26d1c23cdb0070e69db335503dfa378b910eb857386e5b8beba" +dependencies = [ + "bitflags", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cd504cecbfe7eef649988840f9a6cb768d11085efec138f8e99f160632d867" +dependencies = [ + "cc", + "rsqlite-vfs", +] + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toy-rust" +version = "0.1.0" +dependencies = [ + "morca-sdk", + "serde", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/toy-rust/Cargo.toml b/examples/toy-rust/Cargo.toml new file mode 100644 index 00000000..b54d0237 --- /dev/null +++ b/examples/toy-rust/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "toy-rust" +version = "0.1.0" +edition = "2024" +publish = false +description = "examples/toy-rust: examples/toy.mojo rebuilt in Rust with morca-sdk (docs/rust-pipelines.md)." + +[[bin]] +name = "toy_rust" +path = "src/main.rs" + +[dependencies] +morca-sdk = { path = "../../rust/morca-sdk" } +serde = { version = "1", features = ["derive"] } + +[profile.release] +strip = true diff --git a/examples/toy-rust/src/main.rs b/examples/toy-rust/src/main.rs new file mode 100644 index 00000000..a1ce61ac --- /dev/null +++ b/examples/toy-rust/src/main.rs @@ -0,0 +1,81 @@ +//! examples/toy.mojo rebuilt in Rust: three steps over months, scheduled at +//! 03:00 UTC on the 2nd of each month for the month before, from 2024-01, +//! with at most two runs at once. No Mojo compiler and no Mojo runtime. +//! +//! cargo build --release --manifest-path examples/toy-rust/Cargo.toml +//! build/cargo/release/toy_rust describe +//! morca register build/cargo/release/toy_rust +//! +//! As the Mojo toy: a month in MORCA_TOY_FAIL (e.g. `2024-02`) fails its +//! `measure` step, one in MORCA_TOY_CRASH aborts the process there, and +//! MORCA_TOY_SLEEP makes every step take that many seconds. + +use std::time::Duration; + +use morca_sdk::{Cron, Ctx, Month, Pipeline, StepResult, fail}; +use serde::{Deserialize, Serialize}; + +fn nap() { + if let Some(s) = std::env::var("MORCA_TOY_SLEEP") + .ok() + .and_then(|s| s.parse::().ok()) + { + std::thread::sleep(Duration::from_secs_f64(s)); + } +} + +fn listed(var: &str, m: &Month) -> bool { + std::env::var(var) + .unwrap_or_default() + .contains(&m.to_string()) +} + +#[derive(Serialize, Deserialize)] +struct Measured { + month: Month, + days: u32, +} + +/// The number of days in the month. +fn days(_ctx: &mut Ctx, m: Month) -> StepResult { + nap(); + Ok(Measured { + days: m.days(), + month: m, + }) +} + +/// Hours in the month; fails for the months in MORCA_TOY_FAIL. +fn measure(ctx: &mut Ctx, m: Measured) -> StepResult { + nap(); + if listed("MORCA_TOY_FAIL", &m.month) { + return Err(fail(format!("failing {} as asked", m.month))); + } + if listed("MORCA_TOY_CRASH", &m.month) { + eprintln!("crashing {} as asked", m.month); + std::process::abort(); + } + ctx.log(format!("{} has {} days", m.month, m.days)); + Ok(Measured { + days: m.days * 24, + month: m.month, + }) +} + +fn total(ctx: &mut Ctx, ms: Vec) -> StepResult { + nap(); + let t: i64 = ms.iter().map(|m| m.days as i64).sum(); + ctx.log(format!("{} months, {t} hours", ms.len())); + Ok(t) +} + +fn main() { + let toy = Pipeline::::new("toy_rust") + .start("2024-01") + .schedule(Cron::new("0 3 2 * *")) + .max_runs(2) + .mapped("days", days) + .mapped("measure", measure) + .gather("total", total); + morca_sdk::main(toy.build()); +} diff --git a/pixi.toml b/pixi.toml index f292f91d..4328573e 100644 --- a/pixi.toml +++ b/pixi.toml @@ -481,6 +481,10 @@ rust = ">=1.85" [feature.rust.tasks] test-rust-worker = { cmd = "tests/rust_worker.sh", depends-on = ["build-liblake"] } test-hindcast = { cmd = "tests/hindcast.sh", depends-on = ["build-liblake"] } +# morca-sdk (#521): whole pipelines in Rust, with no Mojo toolchain; the +# conformance suite on the Rust toy and a Rust sensor with tables, then the +# Mojo toy and the Rust toy side by side under one scheduler. +test-rust-sdk = { cmd = "scripts/with-postgres.sh tests/rust_sdk.sh" } # MinIO for the tests over object storage (scripts/with-minio.sh). conda-forge # builds it for these two platforms only. diff --git a/rust/morca-sdk/Cargo.lock b/rust/morca-sdk/Cargo.lock new file mode 100644 index 00000000..66cffc41 --- /dev/null +++ b/rust/morca-sdk/Cargo.lock @@ -0,0 +1,4381 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.10.7", + "log", + "miniz_oxide 0.8.9", + "num-bigint", + "quad-rand", + "rand 0.9.5", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum", + "strum_macros", + "thiserror", + "uuid", + "zstd", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrow-arith" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b13b85a6a547a1669114e418f6ab3c3e39c2edb1b8be5ab2889e5739a433ae9" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b11eb61b95868592ad6edbd537465a6940eb3c81a875addecec24283301b267" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53ee609a07eb721c2e4a7669d7bb88e2106c560e7dec1219096fecf5eae356d1" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9575323ffeb4bc6955f39bf8716f734671a32780f1fac9ba6c2891048fc0ea1f" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10791765e6927cdd8ace5bf0e51a69f38a1c3aded2db93561b003c332d02ee2a" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165051f1f996f19b50c4d2cdb7ed02ddd400d8d984b422bf6848507eee099c3d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908a1b5aeefce5c6eb2a0a4e98f4ccacbd441fe0ed23fb799e634cd0135821e5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-schema" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8506ad465ebb9483bb6efa1e2e78a313549122c6dd67dc2f7087297b762eaede" + +[[package]] +name = "arrow-select" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0f0a0d641356ae2df829af5d635ed9fb93fbeb0d39f3239487cb5fdb37a6ef" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd245a358d52e77bfcb99dc3e21f04b947383271b7a0b9ca8feec43e9039234d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "asyncband" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2050892d4691cf110586070786712b504985a4d37ed64a9cd8a402718e1706" + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bon" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a0cf66fd1eb2fb7b822cb78370d82303750809bc6ba855f3f58e517d3238db" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ef9e3e8154a45c7a95cff4c1299702106ad94062d7bb85fdc5d3a5afd91d45" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + +[[package]] +name = "crc32fast" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctor" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea691ed9f36489d087419b8b2ba3ef27d89621380b4178c73e2445a379e4e0f4" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.10", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "dissimilar" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeda16ab4059c5fd2a83f2b9c9e9c981327b18aa8e3b313f7e6563799d4f093e" + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "erased-serde" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e59b668157f32d3459d4bd634861a06ffa2503a461ca5f0916b9f0252605e66" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "expect-test" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63af43ff4431e848fb47472a920f14fa71c24de13255a5692e93d4e90302acb0" +dependencies = [ + "dissimilar", + "once_cell", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "iceberg" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dd5d1aa2fa92d2c9bc426f72bc16717a27eb73255805beb8845afcca31e58d" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "crc32fast", + "derive_builder", + "expect-test", + "fastnum", + "flate2", + "fnv", + "form_urlencoded", + "futures", + "iceberg-property-macro", + "itertools 0.13.0", + "moka", + "murmur3", + "once_cell", + "ordered-float 4.6.0", + "parquet", + "rand 0.9.5", + "reqwest 0.12.28", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd", +] + +[[package]] +name = "iceberg-property-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6de7eea6577fed2290867f7604483e3649297514b873cdac108e0cc7f783c9e1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iceberg-storage-opendal" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08691d92b8e3ce0d827fa3d6e858b6d5fd948fb337bc57c244eb7d6bed1cddbc" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "serde", + "typetag", + "url", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" + +[[package]] +name = "jiff" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175dc66334db3b9ab9985bf8b61cfed50dc16d20964bb47f9e479adcae984a7f" +dependencies = [ + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e599b80292e597019b8b3e392de3f4bea2959ddfb8c7dff60e035e9e311a2c1b" +dependencies = [ + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7b1af7547d47354bc438d797e3854601bde8f8d185780c5ba4ab27afa54f1ba" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0980fca2cc272465bd60e82397607ba9f33add9e48defd77923301bbd03808f2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06e69070b7e12262b0cd90832f0a7396f142a2b330181255562decf9f0c6215" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "link-section" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d9aae57673c1a235149777a74d152f3210ebd3e3b13346d2cdb6809e566bd3c" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lz4_flex" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a027efd2ad3c4dbb4123b5c018e3efbe7f6236da7c7612f122dad030f5d2de5" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "morca-sdk" +version = "0.1.0" +dependencies = [ + "arrow-array", + "arrow-schema", + "async-trait", + "bytes", + "futures", + "iceberg", + "iceberg-storage-opendal", + "object_store", + "parquet", + "postgres", + "rand 0.9.5", + "rusqlite", + "serde", + "serde_json", + "tokio", + "uuid", +] + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + +[[package]] +name = "nix" +version = "0.31.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc81ce8148d3b4f3274da811f11cdd824ae661ee4ef732cdb229825cf5be7f7" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + +[[package]] +name = "object_store" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +dependencies = [ + "async-trait", + "aws-lc-rs", + "base64 0.23.1", + "bytes", + "chrono", + "crc-fast", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools 0.15.0", + "md-5", + "nix", + "parking_lot", + "percent-encoding", + "quick-xml 0.41.0", + "rand 0.10.4", + "reqwest 0.13.5", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +dependencies = [ + "anyhow", + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +dependencies = [ + "asyncband 0.6.7", + "futures", + "http", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-s3" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "58.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b967e8aa77dbc32719087ff91f0f100948a0a7e4c88609bd91b0ccf47e594d7e" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.22.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint", + "num-integer", + "num-traits", + "paste", + "ring", + "seq-macro", + "simdutf8", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "postgres" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ad20e0aa0b24f5a394eab4f78c781d248982b22b25cecc7e3aa46a681605bd" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand 0.10.4", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.10", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.4", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband 0.7.4", + "base64 0.23.1", + "bytes", + "hex", + "hmac", + "http", + "jiff", + "log", + "percent-encoding", + "sha1", + "sha2", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353b74f9810d5b085b6c42c9d86a5f0fdb795553235cdd48753b32e3809d18a7" +dependencies = [ + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f172c4812a19b26d1c23cdb0070e69db335503dfa378b910eb857386e5b8beba" +dependencies = [ + "bitflags", + "fallible-iterator 0.3.0", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "sqlite-wasm-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cd504cecbfe7eef649988840f9a6cb768d11085efec138f8e99f160632d867" +dependencies = [ + "cc", + "rsqlite-vfs", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "thrift" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f650151e44bb59950b3adf32de0064813d1148f34f4884b7f5b539f29596859" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float 3.9.2", + "uuid", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.4", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-normalization" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75cd3fb03d5288b3ba9a1ee103ac7e9cbb59f0b03d64f8ddfcc04dd42758a85d" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993ad287eb20e8ba755312801cfe1413e0f25d604075863acaddca10b04c1833" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] + +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + +[[package]] +name = "uuid" +version = "1.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c14b15dffa2f4bd712103e857140aa04a4f2ff6e955c77d7eb8e53ee02a3b64" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" +dependencies = [ + "cfg-if", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.10", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50adf04cf3ec914b0b9905c9dde90c80fe74bfd701e331fc80a39c512c525278" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498d94ad68720c89b4fbe41b9090850651e68cb2dd9210ce7ffbff3a94653819" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.3+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f729a06d0257428fd9d997ac8db7e1407bd0262ab016b085b36bc52957c0052d" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust/morca-sdk/Cargo.toml b/rust/morca-sdk/Cargo.toml new file mode 100644 index 00000000..b15e3a68 --- /dev/null +++ b/rust/morca-sdk/Cargo.toml @@ -0,0 +1,66 @@ +[package] +name = "morca-sdk" +version = "0.1.0" +edition = "2024" +# iceberg 0.11.0 (the `tables` feature) needs 1.95; the rest builds on less, +# but one floor keeps the crate simple. +rust-version = "1.95" +license = "Apache-2.0" +description = "Write a whole morca pipeline in Rust: steps, partitions and tables, built into a pipeline binary the prebuilt morca scheduler runs (docs/rust-pipelines.md)." +repository = "https://github.com/morca-dev/morca" + +[features] +default = [] +# Iceberg tables in MORCA_WAREHOUSE: reads pinned and recorded, commits +# fenced and carrying the orca.* summary keys, `sense` and `keys`. Pulls in +# iceberg-rust, arrow-rs, parquet and object_store (about 400 crates). +tables = [ + "dep:iceberg", + "dep:iceberg-storage-opendal", + "dep:arrow-array", + "dep:arrow-schema", + "dep:parquet", + "dep:object_store", + "dep:tokio", + "dep:futures", + "dep:bytes", + "dep:async-trait", + "dep:uuid", + "dep:rand", +] + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +# The run stores: Postgres (sync client; no TLS yet) and a SQLite file +# (SQLite compiled in, so the binary needs no libsqlite3). +postgres = "0.19" +rusqlite = { version = "0.43", features = ["bundled"] } + +# The `tables` feature. iceberg 0.11.0 is built on arrow-rs 58, so the crate +# is too: the batches a step hands iceberg must be the same major version. +# morca-worker (rust/morca-worker) is on arrow 60, but the two never share a +# batch: a worker is a cdylib morca loads, a pipeline binary is a program. +iceberg = { version = "=0.11.0", optional = true } +iceberg-storage-opendal = { version = "=0.11.0", default-features = false, features = ["opendal-s3"], optional = true } +arrow-array = { version = "58.4", optional = true } +arrow-schema = { version = "58.4", optional = true } +parquet = { version = "58.4", features = ["async"], optional = true } +object_store = { version = "0.14", features = ["aws"], optional = true } +tokio = { version = "1", features = ["rt-multi-thread", "time"], optional = true } +futures = { version = "0.3", optional = true } +bytes = { version = "1", optional = true } +async-trait = { version = "0.1", optional = true } +uuid = { version = "1", features = ["v4"], optional = true } +rand = { version = "0.9", optional = true } + +[[example]] +name = "clean" +required-features = ["tables"] + +[[example]] +name = "source" +required-features = ["tables"] + +[profile.release] +strip = true diff --git a/rust/morca-sdk/README.md b/rust/morca-sdk/README.md new file mode 100644 index 00000000..c1493b8b --- /dev/null +++ b/rust/morca-sdk/README.md @@ -0,0 +1,46 @@ +# morca-sdk + +Write a whole [morca](https://github.com/morca-dev/morca) pipeline in Rust: +steps, partitions, schedule and Iceberg tables, built with `cargo` into a +pipeline binary that the prebuilt `morca` scheduler registers and runs. No +Mojo compiler and no Mojo runtime. + +```rust +use morca_sdk::{Cron, Ctx, Month, Pipeline, StepResult}; + +fn days(_ctx: &mut Ctx, m: Month) -> StepResult { + Ok(m.days()) +} + +fn total(ctx: &mut Ctx, days: Vec) -> StepResult { + let t = days.iter().sum(); + ctx.log(format!("{t} days")); + Ok(t) +} + +fn main() { + let p = Pipeline::::new("days") + .start("2024-01") + .schedule(Cron::new("0 3 2 * *")) + .mapped("days", days) + .gather("total", total); + morca_sdk::main(p.build()); +} +``` + +```sh +cargo build --release && morca register target/release/days +``` + +- The binary speaks the pipeline-binary protocol, version 1 + (docs/pipeline-protocol.md), for `default` and `control` placement, on + Postgres or a SQLite file, and passes the conformance suite + (tests/conformance.py). +- The `tables` feature adds Iceberg reads (pinned on a retry, recorded) and + appends (fenced, with the `orca.*` summary keys, found again on a retry) + through iceberg-rust 0.11.0 and arrow-rs 58, plus `sense` and `keys`. +- Rust 1.95 or later. + +docs/rust-pipelines.md in the morca repository is the guide, including what +version 1 leaves out. examples/toy-rust is examples/toy.mojo rebuilt with +it; `pixi run -e rust test-rust-sdk` runs both under one scheduler. diff --git a/rust/morca-sdk/examples/clean.rs b/rust/morca-sdk/examples/clean.rs new file mode 100644 index 00000000..77ef011e --- /dev/null +++ b/rust/morca-sdk/examples/clean.rs @@ -0,0 +1,34 @@ +//! A sensor with tables, the Rust twin of tests/sensors/clean.mojo: reads +//! raw.events and appends as many rows to lake.clean, a month at a time. +//! The conformance suite runs it as tests/conformance/clean-rust.json; +//! MORCA_CLEAN_SLEEP makes each attempt take that many seconds first. +//! +//! cargo build --release --features tables --example clean + +use morca_sdk::tables::{id_name_rows, id_name_schema}; +use morca_sdk::{Ctx, Month, Pipeline, StepResult, step}; + +fn clean(ctx: &mut Ctx, m: Month) -> StepResult { + if let Some(s) = std::env::var("MORCA_CLEAN_SLEEP") + .ok() + .and_then(|s| s.parse::().ok()) + { + std::thread::sleep(std::time::Duration::from_secs_f64(s)); + } + let n: usize = ctx.read("raw.events")?.iter().map(|b| b.num_rows()).sum(); + ctx.ensure_table("lake.clean", id_name_schema(), None)?; + ctx.append("lake.clean", vec![id_name_rows(&m.to_string(), n)?])?; + ctx.log(format!("{n} rows for {m}")); + Ok(n as i64) +} + +fn main() { + let p = Pipeline::::new("clean_rust") + .start("2024-01") + .sensor() + .mapped( + step("clean").reads(["raw.events"]).writes(["lake.clean"]), + clean, + ); + morca_sdk::main(p.build()); +} diff --git a/rust/morca-sdk/examples/source.rs b/rust/morca-sdk/examples/source.rs new file mode 100644 index 00000000..ae12cca2 --- /dev/null +++ b/rust/morca-sdk/examples/source.rs @@ -0,0 +1,31 @@ +//! Commits three rows to raw.events as a program that isn't a morca +//! pipeline would, its summary saying which partitions it holds only when +//! given some (the Rust twin of tests/sensors/source.mojo). +//! +//! MORCA_WAREHOUSE= source [] + +use std::collections::HashMap; + +use morca_sdk::tables::{append_as_other, id_name_rows, id_name_schema}; + +fn main() { + let args: Vec = std::env::args().collect(); + let Some(wh) = args.get(1) else { + eprintln!("usage: source []"); + std::process::exit(2); + }; + // SAFETY: single-threaded, before anything reads the environment. + unsafe { std::env::set_var("MORCA_WAREHOUSE", wh) }; + let mut summary = HashMap::new(); + if let Some(p) = args.get(2) { + summary.insert("orca.partitions".to_string(), p.clone()); + } + let rows = id_name_rows("source", 3).unwrap_or_else(|e| panic!("{e}")); + match append_as_other("raw.events", id_name_schema(), vec![rows], summary) { + Ok(snap) => println!("{snap}"), + Err(e) => { + eprintln!("source: {e}"); + std::process::exit(1); + } + } +} diff --git a/rust/morca-sdk/src/cli.rs b/rust/morca-sdk/src/cli.rs new file mode 100644 index 00000000..c2aad1f4 --- /dev/null +++ b/rust/morca-sdk/src/cli.rs @@ -0,0 +1,252 @@ +//! The pipeline binary's command line (docs/pipeline-protocol.md, +//! "Subcommands"): what `default` (local) and `control` placement need. +//! +//! Exit codes: 0 done (for `resume`, the run succeeded); 1 the run failed, +//! deferred or was cancelled, or the heartbeat stopped the process; 2 a +//! usage error; 3 any other error. + +use crate::exec; +use crate::heartbeat::{self, Heartbeat}; +use crate::logs; +use crate::pipeline::Built; +use crate::store::{self, CANCELLED, DEFERRED, DbError, Owned, SUCCEEDED}; + +const USAGE: &str = "usage: describe + resume [--allow-new-code] + sense (a sensor) + keys (a keyed pipeline) + keep-log + put-log + sample ... (runs `morca sample`) +A morca-sdk pipeline binary: the morca scheduler runs it (docs/rust-pipelines.md)."; + +fn usage() -> i32 { + eprintln!("{USAGE}"); + 2 +} + +fn env(k: &str) -> String { + std::env::var(k).unwrap_or_default() +} + +pub(crate) fn run(p: &Built, args: &[String]) -> i32 { + let cmd = args.get(1).map(String::as_str).unwrap_or(""); + if cmd == "sample" { + return sample(&args[2..]); + } + let url = env("MORCA_DATABASE_URL"); + let state_url = env("MORCA_STATE_URL"); + if !url.is_empty() && !state_url.is_empty() { + eprintln!("morca: set MORCA_DATABASE_URL or MORCA_STATE_URL, not both"); + return 2; + } + match (cmd, args.len()) { + ("describe", 2) => { + println!("{}", p.describe()); + 0 + } + ("resume", 3) | ("resume", 4) => { + let allow = match args.get(3).map(String::as_str) { + None => false, + Some("--allow-new-code") => true, + Some(_) => return usage(), + }; + let Ok(run) = args[2].parse::() else { + eprintln!("morca: not a run: {}", args[2]); + return 2; + }; + if !state_url.is_empty() { + eprintln!( + "morca: MORCA_STATE_URL (a worker's run) isn't supported by morca-sdk yet; \ + use default or control placement" + ); + return 2; + } + if url.is_empty() { + eprintln!("morca: resume needs MORCA_DATABASE_URL"); + return 2; + } + resume(p, &url, run, allow) + } + ("sense", 2) => crate::tables::sense(p), + ("keys", 2) => { + if p.cfg.kind != "keyed" { + eprintln!("morca: {} isn't keyed", p.name()); + return 2; + } + crate::tables::keys(p) + } + ("keep-log", 5) => keep_log(&url, &args[2], &args[3], &args[4]), + ("put-log", 4) => match logs::put_log(&args[2], &args[3]) { + Ok(()) => 0, + Err(e) => { + eprintln!("morca: put-log: {e}"); + 3 + } + }, + _ => usage(), + } +} + +/// `sample` is `morca sample`'s: the runner starts it beside each run, and +/// only logs it when it can't. Runs the `morca` on the path. +fn sample(args: &[String]) -> i32 { + match std::process::Command::new("morca") + .arg("sample") + .args(args) + .status() + { + Ok(s) => s.code().unwrap_or(1), + Err(e) => { + eprintln!("morca: sample runs `morca sample`, and there's no morca on the path: {e}"); + 2 + } + } +} + +fn keep_log(url: &str, stage: &str, location: &str, start: &str) -> i32 { + let text = match logs::keep_log(stage, location, start) { + Ok(t) => t, + Err(e) => { + eprintln!("morca: keep-log: {e}"); + return 3; + } + }; + // A driver that crashed left its attempts running: they fail with the + // crash's first frames while the run is running. Exits 0 even when it + // can't record that. + let run = env("MORCA_RUN"); + if let (Some(why), Ok(run), false) = ( + logs::crash_summary(&text), + run.parse::(), + url.is_empty(), + ) { + let r = store::open(url).and_then(|mut s| { + match s.run_info(run)? { + Some(i) if i.status == store::RUNNING => { + let n = s.interrupt(run, &why)?; + println!("run {run} crashed, {n} attempts failed: {why}"); + } + _ => {} + } + Ok(()) + }); + if let Err(e) = r { + println!("morca: can't record run {run} crashing: {e}"); + } + } + 0 +} + +fn resume(p: &Built, url: &str, run: i64, allow_new_code: bool) -> i32 { + match resume_inner(p, url, run, allow_new_code) { + Ok(code) => code, + Err(e) => { + eprintln!("{}", e.to_string().trim_start_matches('\x1f')); + 3 + } + } +} + +fn resume_inner(p: &Built, url: &str, run: i64, allow_new_code: bool) -> Result { + let mut s = store::open(url)?; + s.migrate()?; + let hash = p.registry_hash(); + let Some(info) = s.run_info(run)? else { + return Err(DbError::Final(format!("morca: no run {run}"))); + }; + if info.pipeline != p.name() { + return Err(DbError::Final(format!( + "morca: run {run} is pipeline {}'s, not {}'s", + info.pipeline, + p.name() + ))); + } + if info.registry_hash != hash && !allow_new_code { + return Err(DbError::Final(format!( + "morca: run {run} ran code {}, this binary is {hash}: resume it with \ + --allow-new-code to run it with this code", + info.registry_hash + ))); + } + let mut keys = vec![]; + for k in info.partitions.split(',').filter(|k| !k.is_empty()) { + match (p.key_check)(k) { + Ok(k) => keys.push(k), + Err(e) => { + eprintln!("morca: run {run}: {e}"); + return Ok(2); + } + } + } + let owner = store::process_owner(); + let stale = store::stale_seconds(); + if !s.claim(run, &owner, stale)? { + let i = s.run_info(run)?.unwrap_or(info); + if !["running", "failed", "queued", "deferred"].contains(&i.status.as_str()) { + return Err(DbError::Final(format!("morca: run {run} is {}", i.status))); + } + return Err(DbError::Final(format!( + "morca: run {run} is held by {}, whose heartbeat is fresher than {stale} s", + i.owner + ))); + } + let image = env("MORCA_IMAGE"); + let v = s.version(p.name(), &hash, &p.describe(), &image)?; + s.ran_version(run, v)?; + let interrupted = s.interrupt(run, "")?; + let prior = s.prior(run)?; + println!( + "resuming run {run} with {} steps done and {interrupted} interrupted", + prior.len() + ); + let beater = s.beater(run, &owner, &p.timeouts()); + let mut owned = Owned { + store: s, + run, + owner, + hash, + attempt_id: 0, + }; + let mut hb = Heartbeat::start(beater, run, heartbeat::interval(stale)); + let res = exec::execute(p, &mut owned, &keys, prior); + hb.stop(); + let res = match res { + Ok(r) => r, + Err(e) => { + // A cancel stops the run before its next attempt. + if owned.cancel_requested().unwrap_or(false) { + owned.finish_run(CANCELLED)?; + println!("run {CANCELLED}"); + return Ok(1); + } + return Err(e); + } + }; + let outcome = res.outcome(); + if outcome == DEFERRED { + owned.defer_run(res.deferred_until())?; + } else { + owned.finish_run(outcome)?; + } + for (step, part) in &res.order { + let d = &res.by[&(step.clone(), part.clone())]; + let at = if part.is_empty() { + String::new() + } else { + format!(" {part}") + }; + let err = if d.error.is_empty() { + String::new() + } else { + format!(": {}", d.error) + }; + println!("{step}{at} {}{err}", d.status); + } + if outcome == DEFERRED { + println!("run deferred until {}Z", logs::civil(res.deferred_until())); + } else { + println!("run {outcome}"); + } + Ok(if outcome == SUCCEEDED { 0 } else { 1 }) +} diff --git a/rust/morca-sdk/src/ctx.rs b/rust/morca-sdk/src/ctx.rs new file mode 100644 index 00000000..b03322bd --- /dev/null +++ b/rust/morca-sdk/src/ctx.rs @@ -0,0 +1,215 @@ +//! A step's context: who it is (pipeline, run, step, partition, attempt), +//! its log, progress and metrics, its secrets, the stop flag, and with the +//! `tables` feature its tables (`crate::tables`). morca-worker's +//! `RunContext` has the same calls for a worker step. + +use std::collections::HashMap; +use std::sync::atomic::Ordering; + +use crate::heartbeat; +use crate::logs; +use crate::store::{DbError, Owned}; + +/// What a step gets beside its input. +pub struct Ctx<'a> { + pub(crate) pipeline: String, + pub(crate) run: i64, + pub(crate) step: String, + pub(crate) partition: String, + /// The keys this step's commits hold: its partition, or the run's keys + /// for a gather step, comma-joined. + pub(crate) partitions: String, + pub(crate) attempt: i64, + pub(crate) code: String, + pub(crate) token: String, + /// The snapshot each table (by location) was first read at in this + /// step and partition, by an earlier attempt. + pub(crate) pinned: HashMap, + /// (location, snapshot) of each table this attempt read, in order. + pub(crate) reads: Vec<(String, i64)>, + /// The location of each commit this attempt made, in order. + pub(crate) commits: Vec, + pub(crate) declared_reads: Vec, + pub(crate) declared_writes: Vec, + pub(crate) log_path: Option, + pub(crate) store: &'a mut Owned, +} + +impl Ctx<'_> { + pub fn pipeline(&self) -> &str { + &self.pipeline + } + pub fn run(&self) -> i64 { + self.run + } + pub fn step(&self) -> &str { + &self.step + } + /// The step's partition key; empty for a gather step. + pub fn partition(&self) -> &str { + &self.partition + } + /// 1 for the first attempt of this step and partition in this run. + pub fn attempt(&self) -> i64 { + self.attempt + } + /// The registry hash of the code running. + pub fn code(&self) -> &str { + &self.code + } + + fn tag(&self) -> String { + if self.partition.is_empty() { + format!("[{}] ", self.step) + } else { + format!("[{} {}] ", self.step, self.partition) + } + } + + /// Writes a line tagged with the step and partition to stdout, and with + /// a timestamp to the attempt's log file; secrets masked. + pub fn log(&self, msg: impl AsRef) { + let line = logs::mask(msg.as_ref()); + println!("{}{line}", self.tag()); + self.log_to_file(&line); + } + + pub(crate) fn log_to_file(&self, msg: &str) { + if let Some(p) = &self.log_path { + logs::append_line( + p, + &format!("{} {}{}", logs::utc_now(), self.tag(), logs::mask(msg)), + ); + } + } + + /// Reports progress as a log line: `done` of `total` units. + pub fn progress(&self, done: u64, total: u64) { + self.log(format!("progress {done}/{total}")); + } + + /// Reports a metric as a log line. + pub fn metric(&self, name: &str, value: f64) { + self.log(format!("metric {name}={value}")); + } + + /// A secret the run was given (`MORCA_SECRETS` names it), or None. + pub fn secret(&self, name: &str) -> Option { + if logs::secret_names().iter().any(|n| n == name) { + std::env::var(name).ok() + } else { + None + } + } + + /// Whether the run should end early: it's outdated, since newer code is + /// registered or its scheduler instance drains (Postgres only). Poll it + /// between units of work, from any thread; what to do is up to the step. + pub fn stopping(&self) -> bool { + heartbeat::outdated().flag.load(Ordering::SeqCst) + } + + /// Why [`Ctx::stopping`] is true. + pub fn stopping_why(&self) -> String { + heartbeat::outdated() + .why + .lock() + .map(|w| w.clone()) + .unwrap_or_default() + } + + /// The `orca.*` snapshot summary of this step's next commit to the + /// table at `location` (`Ctx.summary` in src/morca/ctx.mojo). + pub fn summary(&self, location: &str) -> HashMap { + let reads = self + .reads + .iter() + .map(|(t, id)| format!("{t}@{id}")) + .collect::>() + .join(","); + let mut s = HashMap::from([ + ("orca.pipeline".to_string(), self.pipeline.clone()), + ("orca.run".to_string(), self.run.to_string()), + ("orca.step".to_string(), self.step.clone()), + ("orca.partition".to_string(), self.partition.clone()), + ("orca.partitions".to_string(), self.partitions.clone()), + ("orca.attempt".to_string(), self.attempt.to_string()), + ( + "orca.commit".to_string(), + self.commits_to(location).to_string(), + ), + ("orca.code".to_string(), self.code.clone()), + ("orca.reads".to_string(), reads), + ]); + if !self.token.is_empty() { + s.insert("orca.run-token".to_string(), self.token.clone()); + } + s + } + + pub(crate) fn commits_to(&self, location: &str) -> usize { + self.commits.iter().filter(|t| *t == location).count() + } + + /// `Ctx.made`: whether a snapshot with `summary` is this step's next + /// commit to the table at `location`, by any attempt of this run. Every + /// key must be there; a run token, when both have one, must agree. + pub fn made(&self, location: &str, summary: &HashMap) -> bool { + if self.run <= 0 { + return false; + } + let has = |k: &str, v: &str| summary.get(k).is_some_and(|x| x == v); + let token_fits = self.token.is_empty() + || summary + .get("orca.run-token") + .is_none_or(|t| t.is_empty() || *t == self.token); + has("orca.pipeline", &self.pipeline) + && has("orca.run", &self.run.to_string()) + && has("orca.step", &self.step) + && has("orca.partition", &self.partition) + && has("orca.commit", &self.commits_to(location).to_string()) + && token_fits + } + + /// The snapshot to read the table at `location` at: the one this + /// attempt already read, else the one an earlier attempt read, else + /// `current` (-1 for an empty table). + pub(crate) fn pin(&self, location: &str, current: i64) -> i64 { + if let Some((_, s)) = self.reads.iter().find(|(t, _)| t == location) { + return *s; + } + self.pinned.get(location).copied().unwrap_or(current) + } + + /// Records a read of `location` at `snapshot`, once per attempt. + pub(crate) fn record_read( + &mut self, + location: &str, + snapshot: i64, + dataset: &str, + ) -> Result<(), DbError> { + if self.reads.iter().any(|(t, _)| t == location) { + return Ok(()); + } + self.store.record(false, location, snapshot, dataset)?; + self.reads.push((location.to_string(), snapshot)); + Ok(()) + } + + /// Records this attempt's commit of `snapshot` to `location`. + pub(crate) fn record_commit( + &mut self, + location: &str, + snapshot: i64, + dataset: &str, + ) -> Result<(), DbError> { + self.store.record(true, location, snapshot, dataset)?; + self.commits.push(location.to_string()); + Ok(()) + } + + /// The commit fence: whether this process may still commit. + pub(crate) fn fence(&mut self) -> Result<(), DbError> { + self.store.fence() + } +} diff --git a/rust/morca-sdk/src/exec.rs b/rust/morca-sdk/src/exec.rs new file mode 100644 index 00000000..48c46637 --- /dev/null +++ b/rust/morca-sdk/src/exec.rs @@ -0,0 +1,297 @@ +//! Executing a claimed run (docs/pipeline-protocol.md, "Executing"): each +//! step in describe order, each key for a mapped step or once for a gather +//! step, skipping what succeeded before, skipping what follows a step that +//! didn't succeed, and trying each attempt up to the step's retries. + +use std::collections::HashMap; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use crate::ctx::Ctx; +use crate::logs; +use crate::pipeline::{Built, Kind, StepDef, StepError, decode_value}; +use crate::store::{DEFERRED, FAILED, Owned, Res, SKIPPED, SUCCEEDED}; + +/// One step and partition's outcome in this run. +pub(crate) struct Done { + pub status: &'static str, + pub output: Vec, + pub error: String, + /// When a deferred step may run again (Unix seconds). + pub until: i64, +} + +pub(crate) struct Results { + /// (step, partition) in the order they were settled. + pub order: Vec<(String, String)>, + pub by: HashMap<(String, String), Done>, +} + +impl Results { + fn ok(&self, step: &str, partition: &str) -> Option<&Vec> { + self.by + .get(&(step.to_string(), partition.to_string())) + .filter(|d| d.status == SUCCEEDED) + .map(|d| &d.output) + } + + fn put(&mut self, step: &str, partition: &str, d: Done) { + let k = (step.to_string(), partition.to_string()); + if !self.by.contains_key(&k) { + self.order.push(k.clone()); + } + self.by.insert(k, d); + } + + /// `succeeded` if every step did, else `deferred` if one deferred and + /// none failed, else `failed`. + pub fn outcome(&self) -> &'static str { + let all: Vec<&Done> = self.by.values().collect(); + if all.iter().all(|d| d.status == SUCCEEDED) { + SUCCEEDED + } else if all.iter().any(|d| d.status == DEFERRED) + && !all.iter().any(|d| d.status == FAILED) + { + DEFERRED + } else { + FAILED + } + } + + pub fn deferred_until(&self) -> i64 { + self.by.values().map(|d| d.until).max().unwrap_or(0) + } +} + +fn panic_text(p: Box) -> String { + if let Some(s) = p.downcast_ref::<&str>() { + s.to_string() + } else if let Some(s) = p.downcast_ref::() { + s.clone() + } else { + "a panic".to_string() + } +} + +/// Runs what's left of the run over `keys`. A store error is returned: the +/// run stops where it is and can be resumed. +pub(crate) fn execute( + p: &Built, + store: &mut Owned, + keys: &[String], + prior: HashMap<(String, String), Vec>, +) -> Res { + let mut res = Results { + order: vec![], + by: HashMap::new(), + }; + for ((s, part), out) in prior { + res.put( + &s, + &part, + Done { + status: SUCCEEDED, + output: out, + error: String::new(), + until: 0, + }, + ); + } + let all_keys = keys.join(","); + for (i, st) in p.steps.iter().enumerate() { + let name = st.spec.name.as_str(); + let up = st.after.as_deref(); + let up_kind = i.checked_sub(1).map(|j| p.steps[j].kind); + match st.kind { + Kind::Mapped => { + for key in keys { + if res.ok(name, key).is_some() { + continue; + } + let (input, blocked) = match up { + None => ( + (p.key_input)(key).map_err(crate::store::DbError::Retry)?, + None, + ), + Some(u) => match res.ok(u, key) { + Some(o) => (o.clone(), None), + None => (vec![], Some(format!("{u} {key}"))), + }, + }; + run_step(p, st, i, store, &mut res, key, input, blocked, key)?; + } + } + Kind::Gather => { + if res.ok(name, "").is_some() { + continue; + } + let (input, blocked) = match (up, up_kind) { + (None, _) => ( + (p.keys_input)(keys).map_err(crate::store::DbError::Retry)?, + None, + ), + (Some(u), Some(Kind::Gather)) => match res.ok(u, "") { + Some(o) => (o.clone(), None), + None => (vec![], Some(u.to_string())), + }, + (Some(u), _) => gather_input(p, st, i, &res, u, keys), + }; + run_step(p, st, i, store, &mut res, "", input, blocked, &all_keys)?; + } + } + } + Ok(res) +} + +/// A gather's input over a mapped step: the upstream outputs in key order, +/// each checked to be the upstream's output type; or what blocks it. +fn gather_input( + p: &Built, + st: &StepDef, + i: usize, + res: &Results, + up: &str, + keys: &[String], +) -> (Vec, Option) { + let want = &p.steps[i - 1].output; + let mut blocked = vec![]; + let mut values = vec![]; + for k in keys { + match res.ok(up, k) { + None => blocked.push(format!("{up} {k}")), + Some(o) => match decode_value(o, want) { + Ok(v) => values.push(v), + // An output an older build stored as another type: the + // step's decode then fails, naming it. + Err(e) => { + let bad = json!({"type": format!("{up} [{k}]'s output ({e})"), "value": null}); + return (serde_json::to_vec(&bad).unwrap(), None); + } + }, + } + } + if !blocked.is_empty() { + return (vec![], Some(blocked.join(", "))); + } + let input = json!({"type": st.input, "value": Value::Array(values)}); + (serde_json::to_vec(&input).unwrap(), None) +} + +#[allow(clippy::too_many_arguments)] +fn run_step( + p: &Built, + st: &StepDef, + ord: usize, + store: &mut Owned, + res: &mut Results, + partition: &str, + input: Vec, + blocked: Option, + holds: &str, +) -> Res<()> { + let name = st.spec.name.as_str(); + if let Some(b) = blocked { + let reason = format!("not run: {b} didn't succeed"); + store.skip(name, partition, ord as i32, &reason)?; + res.put( + name, + partition, + Done { + status: SKIPPED, + output: vec![], + error: reason, + until: 0, + }, + ); + return Ok(()); + } + // Every attempt reads the snapshots the first one read. + let mut pinned = store.pinned_reads(name, partition)?; + let token = store.run_token()?; + let code = p.registry_hash(); + let mut wait = st.spec.backoff_seconds as f64; + let mut outcome = Done { + status: FAILED, + output: vec![], + error: String::new(), + until: 0, + }; + for k in 0..=st.spec.retries { + if k > 0 && wait > 0.0 { + println!("[{name} {partition}] retrying in {} s", wait as i64); + std::thread::sleep(Duration::from_secs_f64(wait)); + wait *= 2.0; + } + let attempt = store.start_attempt(name, partition, ord as i32, &input)?; + let log = logs::attempt_log(p.name(), store.run, name, partition, attempt); + if let Some(l) = &log { + store.set_log(l)?; + } + let mut ctx = Ctx { + pipeline: p.name().to_string(), + run: store.run, + step: name.to_string(), + partition: partition.to_string(), + partitions: holds.to_string(), + attempt, + code: code.clone(), + token: token.clone(), + pinned: pinned.clone(), + reads: vec![], + commits: vec![], + declared_reads: st.spec.reads.clone(), + declared_writes: st.spec.writes.clone(), + log_path: log, + store, + }; + ctx.log_to_file(&format!("attempt {attempt} started")); + let got = match catch_unwind(AssertUnwindSafe(|| (st.run)(&mut ctx, &input))) { + Ok(r) => r, + Err(panic) => Err(StepError::Failed(format!( + "panicked: {}", + panic_text(panic) + ))), + }; + match &got { + Ok(_) => ctx.log_to_file(&format!("attempt {attempt} succeeded")), + Err(e) => ctx.log(format!("attempt {attempt} failed: {}", e.message())), + } + for (t, s) in &ctx.reads { + pinned.entry(t.clone()).or_insert(*s); + } + drop(ctx); + match got { + Ok(out) => { + store.finish_attempt(name, partition, attempt, SUCCEEDED, &out, "")?; + outcome = Done { + status: SUCCEEDED, + output: out, + error: String::new(), + until: 0, + }; + break; + } + Err(StepError::Defer { seconds, why }) => { + let until = logs::unix_now() + seconds as i64; + let why = format!("deferred until {}Z: {why}", logs::civil(until)); + store.finish_attempt(name, partition, attempt, DEFERRED, &[], &why)?; + outcome = Done { + status: DEFERRED, + output: vec![], + error: why, + until, + }; + break; + } + Err(StepError::Failed(e)) => { + let e = logs::mask(&e); + store.finish_attempt(name, partition, attempt, FAILED, &[], &e)?; + outcome.error = e; + } + } + } + res.put(name, partition, outcome); + Ok(()) +} diff --git a/rust/morca-sdk/src/heartbeat.rs b/rust/morca-sdk/src/heartbeat.rs new file mode 100644 index 00000000..317462d0 --- /dev/null +++ b/rust/morca-sdk/src/heartbeat.rs @@ -0,0 +1,121 @@ +//! The heartbeat (docs/pipeline-protocol.md, "Heartbeat"): while a run's +//! steps run, a thread of its own, on a connection of its own, refreshes the +//! run's heartbeat every `MORCA_STALE_SECONDS / 5`, runs the watchdog (a +//! step past its `timeout_seconds`, a cancel whose `cancel_at` has passed) +//! and asks whether the run is outdated. A failed beat drops the connection +//! and tries again on the next one. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// One beat on the heartbeat's connection. +pub(crate) trait Beater: Send { + /// Refreshes the heartbeat and runs the watchdog: why the process must + /// stop now, already recorded, or None. + fn beat(&mut self) -> Result, String>; + /// Drops the connection, for the next beat to open another. + fn reset(&mut self); +} + +/// Whether the run is outdated, and why: the stop flag steps read. +#[derive(Default)] +pub(crate) struct Outdated { + pub flag: AtomicBool, + pub why: Mutex, +} + +pub(crate) fn outdated() -> &'static Outdated { + static O: OnceLock = OnceLock::new(); + O.get_or_init(Outdated::default) +} + +pub(crate) fn set_outdated(o: bool, why: String) { + let st = outdated(); + st.flag.store(o, Ordering::SeqCst); + if let Ok(mut w) = st.why.lock() { + *w = why; + } +} + +/// Ends the process at once, as the Mojo binary's `_exit(1)`: no +/// destructors, no other thread finishing what it was doing. +pub(crate) fn exit_now(code: i32) -> ! { + unsafe extern "C" { + fn _exit(code: std::ffi::c_int) -> !; + } + use std::io::Write; + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); + unsafe { _exit(code) } +} + +pub(crate) struct Heartbeat { + stop: Arc, + thread: Option>, +} + +impl Heartbeat { + pub fn start(mut beater: Box, run: i64, interval: f64) -> Heartbeat { + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let every = Duration::from_secs_f64(interval.max(0.05)); + let thread = std::thread::Builder::new() + .name("morca-heartbeat".into()) + .spawn(move || { + let mut next = Instant::now(); + while !flag.load(Ordering::SeqCst) { + if Instant::now() >= next { + next = Instant::now() + every; + match beater.beat() { + Ok(Some(why)) => { + println!("run {run} {why}"); + exit_now(1); + } + Ok(None) => {} + Err(e) => { + eprintln!("morca: heartbeat: {e}"); + beater.reset(); + } + } + } + std::thread::sleep(Duration::from_millis(20).min(every)); + } + }) + .expect("morca: can't start the heartbeat thread"); + Heartbeat { + stop, + thread: Some(thread), + } + } + + pub fn stop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(t) = self.thread.take() { + let _ = t.join(); + } + } +} + +impl Drop for Heartbeat { + fn drop(&mut self) { + self.stop(); + } +} + +/// The interval: five beats per stale limit. +pub(crate) fn interval(stale: f64) -> f64 { + stale / 5.0 +} + +/// The watchdog's error for an attempt past its step's timeout. +pub(crate) fn timed_out_why(secs: f64) -> String { + format!( + "timed out after {} s (timeout_seconds), so its process was stopped", + secs as i64 + ) +} + +pub(crate) const CANCEL_WHY: &str = "cancelled: its step hadn't finished when the cancel's \ +MORCA_CANCEL_KILL_SECONDS ran out, so its process was stopped"; diff --git a/rust/morca-sdk/src/lib.rs b/rust/morca-sdk/src/lib.rs new file mode 100644 index 00000000..f44a58c1 --- /dev/null +++ b/rust/morca-sdk/src/lib.rs @@ -0,0 +1,79 @@ +//! Write a whole morca pipeline in Rust: steps, partitions and tables, +//! built into a pipeline binary that the prebuilt `morca` scheduler +//! registers and runs like any other, with no Mojo toolchain. +//! +//! The binary speaks the pipeline-binary protocol (docs/pipeline-protocol.md +//! in the morca repository), version 1, for `default` (local) and `control` +//! placement: `describe`, `resume`, `sense`, `keys`, `keep-log`, `put-log`, +//! on Postgres or a SQLite file. docs/rust-pipelines.md is the guide. +//! +//! ```no_run +//! use morca_sdk::{Ctx, Month, Pipeline, StepResult, Cron}; +//! use serde::{Deserialize, Serialize}; +//! +//! #[derive(Serialize, Deserialize)] +//! struct Measured { month: Month, days: u32 } +//! +//! fn days(_ctx: &mut Ctx, m: Month) -> StepResult { +//! Ok(Measured { days: m.days(), month: m }) +//! } +//! +//! fn total(ctx: &mut Ctx, ms: Vec) -> StepResult { +//! let t = ms.iter().map(|m| m.days as u64 * 24).sum(); +//! ctx.log(format!("{} months, {t} hours", ms.len())); +//! Ok(t) +//! } +//! +//! fn main() { +//! let p = Pipeline::::new("hours") +//! .start("2024-01") +//! .schedule(Cron::new("0 3 2 * *")) +//! .mapped("days", days) +//! .gather("total", total); +//! morca_sdk::main(p.build()); +//! } +//! ``` + +// Without `tables`, the read and commit plumbing has no caller. +#![cfg_attr(not(feature = "tables"), allow(dead_code))] + +mod cli; +mod ctx; +mod exec; +mod heartbeat; +mod logs; +mod partition; +mod pipeline; +mod store; +pub mod tables; + +pub use ctx::Ctx; +pub use partition::{Day, Hour, Month, Name, Partition, Week, days_in_month}; +pub use pipeline::{ + AfterGather, AfterMapped, Built, Cron, Pipeline, Source, Start, StepError, StepResult, + StepSpec, defer, fail, step, +}; +pub use serde; +pub use serde_json; + +/// The version of the pipeline-binary protocol this crate speaks: +/// describe's `"protocol"`. +pub const PROTOCOL_VERSION: u32 = 1; + +/// Runs the binary's command line for the pipeline, and exits with its +/// code. A definition that didn't build is a usage error (exit 2), so a +/// broken binary can't be registered. +pub fn main(pipeline: Result) -> ! { + let p = match pipeline { + Ok(p) => p, + Err(e) => { + eprintln!("{e}"); + std::process::exit(2); + } + }; + let args: Vec = std::env::args().collect(); + let code = cli::run(&p, &args); + use std::io::Write; + let _ = std::io::stdout().flush(); + std::process::exit(code) +} diff --git a/rust/morca-sdk/src/logs.rs b/rust/morca-sdk/src/logs.rs new file mode 100644 index 00000000..2fca9e46 --- /dev/null +++ b/rust/morca-sdk/src/logs.rs @@ -0,0 +1,326 @@ +//! Logs (docs/pipeline-protocol.md, "Logs" and "keep-log"): each attempt's +//! log at `/////.log`, +//! secrets masked, and what `keep-log` finds in a driver's output when it +//! crashed. +//! +//! Locations are on this machine: a directory or `file://`. A `MORCA_LOGS` +//! in object storage isn't supported yet; attempts then keep no log file. + +use std::io::Write; +use std::path::Path; + +use crate::partition::path_segment; + +const MASK: &str = "***"; + +/// The values of the variables `MORCA_SECRETS` names; short ones are left +/// out, since masking them would hide ordinary text. +pub(crate) fn secret_values() -> Vec { + secret_names() + .iter() + .filter_map(|n| std::env::var(n).ok()) + .filter(|v| v.len() >= 4) + .collect() +} + +pub(crate) fn secret_names() -> Vec { + std::env::var("MORCA_SECRETS") + .unwrap_or_default() + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect() +} + +/// `text` with every secret's value replaced by `***`. +pub(crate) fn mask(text: &str) -> String { + let mut out = text.to_string(); + for v in secret_values() { + out = out.replace(&v, MASK); + } + out +} + +pub(crate) fn is_remote(location: &str) -> bool { + location.contains("://") && !location.starts_with("file://") +} + +pub(crate) fn local_path(location: &str) -> &str { + location.strip_prefix("file://").unwrap_or(location) +} + +/// The run's directory under `MORCA_LOGS`, or None without it (or in +/// object storage, not supported yet). +pub(crate) fn run_dir(pipeline: &str, run: i64) -> Option { + let root = std::env::var("MORCA_LOGS").ok().filter(|s| !s.is_empty())?; + if is_remote(&root) { + eprintln!( + "morca: MORCA_LOGS in object storage isn't supported by morca-sdk yet: no attempt logs" + ); + return None; + } + Some(format!("{}/{pipeline}/{run}", root.trim_end_matches('/'))) +} + +/// Where the attempt's log goes, its directory made; None without +/// `MORCA_LOGS`. +pub(crate) fn attempt_log( + pipeline: &str, + run: i64, + step: &str, + partition: &str, + attempt: i64, +) -> Option { + let dir = format!( + "{}/{step}/{}", + run_dir(pipeline, run)?, + if partition.is_empty() { + "_run".to_string() + } else { + path_segment(partition) + } + ); + if let Err(e) = std::fs::create_dir_all(local_path(&dir)) { + eprintln!("morca: can't make {dir}: {e}"); + return None; + } + Some(format!("{dir}/{attempt}.log")) +} + +/// Appends `line` to the file at `location`. +pub(crate) fn append_line(location: &str, line: &str) { + let r = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(local_path(location)) + .and_then(|mut f| writeln!(f, "{line}")); + if let Err(e) = r { + eprintln!("morca: can't write {location}: {e}"); + } +} + +/// Civil time of Unix seconds, `YYYY-MM-DDTHH:MM:SS`. +pub(crate) fn civil(secs: i64) -> String { + let days = secs.div_euclid(86400); + let rem = secs.rem_euclid(86400); + // Howard Hinnant's civil_from_days. + let z = days + 719468; + let era = z.div_euclid(146097); + let doe = z - era * 146097; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!( + "{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}", + rem / 3600, + rem % 3600 / 60, + rem % 60 + ) +} + +pub(crate) fn unix_now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// The time now, to the second, as `YYYY-MM-DDTHH:MM:SSZ`. +pub(crate) fn utc_now() -> String { + format!("{}Z", civil(unix_now())) +} + +fn read_location(location: &str) -> std::io::Result> { + if is_remote(location) { + return Err(std::io::Error::other(format!( + "{location}: object storage isn't supported by morca-sdk yet" + ))); + } + std::fs::read(local_path(location)) +} + +/// `keep-log`'s copy: appends `--- start ` and the stage file to +/// `location`, masked; returns the stage file's text. +pub(crate) fn keep_log(stage: &str, location: &str, start: &str) -> std::io::Result { + let data = read_location(stage)?; + if is_remote(location) { + return Err(std::io::Error::other(format!( + "{location}: object storage isn't supported by morca-sdk yet" + ))); + } + let text = String::from_utf8_lossy(&data).into_owned(); + let path = local_path(location); + if let Some(dir) = Path::new(path).parent() + && !dir.as_os_str().is_empty() + { + std::fs::create_dir_all(dir)?; + } + let mut f = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + write!(f, "--- start {start}\n{}", mask(&text))?; + Ok(text) +} + +/// `put-log`: copies a local file to `location`, replacing it. +pub(crate) fn put_log(local: &str, location: &str) -> std::io::Result<()> { + let data = std::fs::read(local)?; + if is_remote(location) { + return Err(std::io::Error::other(format!( + "{location}: object storage isn't supported by morca-sdk yet" + ))); + } + let path = local_path(location); + let tmp = format!("{path}.tmp-{}", std::process::id()); + std::fs::write(&tmp, data)?; + std::fs::rename(&tmp, path) +} + +const FRAMES: usize = 4; +const LIMIT: usize = 300; + +fn clip(s: &str) -> String { + if s.len() <= LIMIT { + return s.to_string(); + } + let mut n = LIMIT; + while !s.is_char_boundary(n) { + n -= 1; + } + format!("{}...", &s[..n]) +} + +/// `crashed: ; at , ...` for the last crash +/// in `output` after its last `--- start` line, or None: a Mojo stack dump +/// (src/morca/_internal/crash.mojo), which a Mojo pipeline's driver output +/// holds, or a Rust panic that ended the process (`panic = "abort"`, or a +/// panic outside a step). +pub(crate) fn crash_summary(output: &str) -> Option { + let lines: Vec<&str> = output.lines().collect(); + let mut first = 0; + for (i, l) in lines.iter().enumerate() { + if l.starts_with("--- start ") { + first = i + 1; + } + } + let mut dump: Option<(usize, bool)> = None; + let mut panic: Option = None; + for (i, l) in lines.iter().enumerate().skip(first) { + let l = l.trim(); + if l.starts_with("#0 0x") { + dump = Some((i, true)); + } else if l.starts_with("Stack dump without symbol names") { + dump = Some((i, false)); + } else if l.starts_with("thread '") && l.contains("' panicked at ") { + panic = Some(i); + } + } + if let Some((at, symbols)) = dump + && panic.is_none_or(|p| p < at) + { + let mut out = "crashed".to_string(); + if at > first { + let said = lines[at - 1].trim(); + let cut = said + .strip_prefix("ABORT: ") + .or_else(|| said.strip_prefix("At: ")); + if let Some(c) = cut { + out.push_str(&format!(": {}", clip(c))); + } + } + if !symbols { + out.push_str( + "; its stack dump has no symbol names (no llvm-symbolizer: set LLVM_SYMBOLIZER_PATH)", + ); + return Some(out); + } + let mut named = vec![]; + for l in &lines[at..] { + let l = l.trim(); + if !l.starts_with('#') { + break; + } + let parts: Vec<&str> = l.splitn(3, ' ').collect(); + if parts.len() < 3 { + continue; + } + let rest = parts[2]; + let Some(sp) = rest.rfind(' ') else { continue }; + if rest.starts_with('(') { + continue; + } + let at = &rest[sp + 1..]; + if at.ends_with(')') || at.starts_with("Mojo/stdlib/") { + continue; + } + let mut end = sp; + for c in ['[', '('] { + if let Some(k) = rest.find(c) + && k > 0 + && k < end + { + end = k; + } + } + named.push(format!("{} {at}", &rest[..end])); + if named.len() == FRAMES { + break; + } + } + if named.is_empty() { + out.push_str("; its stack dump has no line numbers (build with mojo build -g1)"); + } else { + out.push_str(&format!("; at {}", named.join(", "))); + } + return Some(out); + } + let p = panic?; + // `thread 'main' panicked at src/main.rs:9:5:` then the message. + let head = lines[p].trim(); + let at = head + .split("' panicked at ") + .nth(1) + .unwrap_or("") + .trim_end_matches(':'); + let msg = lines.get(p + 1).map(|s| s.trim()).unwrap_or(""); + Some(format!("crashed: {}; at {at}", clip(msg))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mojo_dump() { + let out = "--- start 1\nABORT: /p/c2.mojo:6:14: the input is bad\n #0 0x0000000102be67b8 (/x/lib/libKGEN.dylib+0xba7b8)\n #2 0x000000010286863c _abort_base Mojo/stdlib/std/os/os.mojo:242:31\n #4 0x000000010286863c main /p/c2.mojo:6:14\n"; + assert_eq!( + crash_summary(out).unwrap(), + "crashed: /p/c2.mojo:6:14: the input is bad; at main /p/c2.mojo:6:14" + ); + } + + #[test] + fn rust_panic() { + let out = + "thread 'main' panicked at src/main.rs:9:5:\nboom\nnote: run with RUST_BACKTRACE=1\n"; + assert_eq!( + crash_summary(out).unwrap(), + "crashed: boom; at src/main.rs:9:5" + ); + assert!(crash_summary("all fine\n").is_none()); + assert!( + crash_summary("thread 'main' panicked at x:1:1:\nboom\n--- start 2\nok\n").is_none() + ); + } + + #[test] + fn civil_time() { + assert_eq!(civil(0), "1970-01-01T00:00:00"); + assert_eq!(civil(1709251200), "2024-03-01T00:00:00"); + } +} diff --git a/rust/morca-sdk/src/partition.rs b/rust/morca-sdk/src/partition.rs new file mode 100644 index 00000000..77c4724b --- /dev/null +++ b/rust/morca-sdk/src/partition.rs @@ -0,0 +1,301 @@ +//! Partition keys: the text the scheduler and the binary both make and +//! parse (docs/pipeline-protocol.md, "Partition keys"). `monthly` is +//! `YYYY-MM`, `weekly` `YYYY-Www` (ISO weeks), `daily` `YYYY-MM-DD`, `hourly` +//! `YYYY-MM-DDTHH`, and `keyed` the names the keys dataset holds. +//! +//! A key is serialised as its text, so a first step's input row +//! (`orca.steps.input`) reads as the key. + +use std::fmt; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +/// A partition key type: one of [`Month`], [`Week`], [`Day`], [`Hour`] or +/// [`Name`]. +pub trait Partition: + Clone + fmt::Display + Serialize + DeserializeOwned + Send + Sync + 'static +{ + /// describe's `partitions.kind`. + const KIND: &'static str; + /// Parses a key as the scheduler writes it. + fn parse(s: &str) -> Result; +} + +fn num(s: &str, what: &str, key: &str) -> Result { + if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) { + return Err(format!("not a {what}: '{key}'")); + } + s.parse().map_err(|_| format!("not a {what}: '{key}'")) +} + +fn leap(y: u32) -> bool { + (y.is_multiple_of(4) && !y.is_multiple_of(100)) || y.is_multiple_of(400) +} + +/// The number of days in a month. +pub fn days_in_month(year: u32, month: u32) -> u32 { + match month { + 2 if leap(year) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +macro_rules! text_serde { + ($t:ty) => { + impl Serialize for $t { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.to_string()) + } + } + impl<'de> Deserialize<'de> for $t { + fn deserialize>(d: D) -> Result { + let s = String::deserialize(d)?; + <$t as Partition>::parse(&s).map_err(serde::de::Error::custom) + } + } + }; +} + +/// A month, `YYYY-MM`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Month { + pub year: u32, + pub month: u32, +} + +impl Month { + pub fn days(&self) -> u32 { + days_in_month(self.year, self.month) + } +} + +impl fmt::Display for Month { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:04}-{:02}", self.year, self.month) + } +} + +impl Partition for Month { + const KIND: &'static str = "monthly"; + fn parse(s: &str) -> Result { + let (y, m) = s + .split_once('-') + .filter(|(y, m)| y.len() == 4 && m.len() == 2) + .ok_or_else(|| format!("not a month (YYYY-MM): '{s}'"))?; + let (year, month) = (num(y, "year", s)?, num(m, "month", s)?); + if !(1..=12).contains(&month) { + return Err(format!("not a month (YYYY-MM): '{s}'")); + } + Ok(Month { year, month }) + } +} +text_serde!(Month); + +/// A day, `YYYY-MM-DD`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Day { + pub year: u32, + pub month: u32, + pub day: u32, +} + +impl fmt::Display for Day { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:04}-{:02}-{:02}", self.year, self.month, self.day) + } +} + +impl Partition for Day { + const KIND: &'static str = "daily"; + fn parse(s: &str) -> Result { + let bad = || format!("not a day (YYYY-MM-DD): '{s}'"); + if s.len() != 10 || &s[4..5] != "-" || &s[7..8] != "-" { + return Err(bad()); + } + let m = Month::parse(&s[..7]).map_err(|_| bad())?; + let day = num(&s[8..], "day", s)?; + if day < 1 || day > m.days() { + return Err(bad()); + } + Ok(Day { + year: m.year, + month: m.month, + day, + }) + } +} +text_serde!(Day); + +/// An hour, `YYYY-MM-DDTHH`, in UTC. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Hour { + pub day: Day, + pub hour: u32, +} + +impl fmt::Display for Hour { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}T{:02}", self.day, self.hour) + } +} + +impl Partition for Hour { + const KIND: &'static str = "hourly"; + fn parse(s: &str) -> Result { + let bad = || format!("not an hour (YYYY-MM-DDTHH): '{s}'"); + let (d, h) = s.split_once('T').ok_or_else(bad)?; + let day = Day::parse(d).map_err(|_| bad())?; + if h.len() != 2 { + return Err(bad()); + } + let hour = num(h, "hour", s)?; + if hour > 23 { + return Err(bad()); + } + Ok(Hour { day, hour }) + } +} +text_serde!(Hour); + +/// An ISO week, `YYYY-Www`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Week { + pub year: u32, + pub week: u32, +} + +/// Days since 1970-01-01 of a civil date. +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let yy = if m <= 2 { y - 1 } else { y }; + let era = yy.div_euclid(400); + let yoe = yy - era * 400; + let mp = if m > 2 { m - 3 } else { m + 9 }; + let doy = (153 * mp + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +/// How many ISO weeks a year has: 53 when 1 January is a Thursday, or a +/// Wednesday in a leap year. +fn iso_weeks(year: u32) -> u32 { + // 1970-01-01 was a Thursday: weekday 0 = Monday. + let wd = (days_from_civil(year as i64, 1, 1) + 3).rem_euclid(7); + if wd == 3 || (wd == 2 && leap(year)) { + 53 + } else { + 52 + } +} + +impl fmt::Display for Week { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:04}-W{:02}", self.year, self.week) + } +} + +impl Partition for Week { + const KIND: &'static str = "weekly"; + fn parse(s: &str) -> Result { + let bad = || format!("not a week (YYYY-Www): '{s}'"); + if s.len() != 8 || &s[4..6] != "-W" { + return Err(bad()); + } + let year = num(&s[..4], "year", s)?; + let week = num(&s[6..], "week", s)?; + if week < 1 || week > iso_weeks(year) { + return Err(bad()); + } + Ok(Week { year, week }) + } +} +text_serde!(Week); + +/// The partition key of a keyed pipeline: 1 to 128 bytes of +/// `[A-Za-z0-9._:/-]`, no `..`, and no `/` at either end or twice in a row +/// (src/morca/name.mojo). +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Name(pub String); + +impl Name { + /// The name as one segment of a path: each `/` written `%2F`. + pub fn path_segment(&self) -> String { + self.0.replace('/', "%2F") + } +} + +impl fmt::Display for Name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Partition for Name { + const KIND: &'static str = "keyed"; + fn parse(s: &str) -> Result { + let ok = !s.is_empty() + && s.len() <= 128 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._:-/".contains(&b)) + && !s.contains("..") + && !s.contains("//") + && !s.starts_with('/') + && !s.ends_with('/'); + if !ok { + return Err(format!( + "not a name (1 to 128 bytes of A-Z a-z 0-9 . _ : - /): '{s}'" + )); + } + Ok(Name(s.to_string())) + } +} +text_serde!(Name); + +/// A key as one segment of an attempt's log path. +pub(crate) fn path_segment(key: &str) -> String { + key.replace('/', "%2F") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_round_trip() { + for s in ["2024-01", "1999-12"] { + assert_eq!(Month::parse(s).unwrap().to_string(), s); + } + for s in ["2024-02-29", "2023-12-31"] { + assert_eq!(Day::parse(s).unwrap().to_string(), s); + } + assert_eq!( + Hour::parse("2024-03-01T07").unwrap().to_string(), + "2024-03-01T07" + ); + assert_eq!(Week::parse("2020-W53").unwrap().to_string(), "2020-W53"); + assert_eq!(Name::parse("owner/repo").unwrap().to_string(), "owner/repo"); + } + + #[test] + fn bad_keys() { + for s in ["2024-13", "2024-1", "24-01", "2024-00", "x"] { + assert!(Month::parse(s).is_err(), "{s}"); + } + assert!(Day::parse("2023-02-29").is_err()); + assert!(Hour::parse("2024-03-01T24").is_err()); + assert!(Week::parse("2021-W53").is_err()); + for s in ["", "a..b", "/a", "a/", "a//b", "a,b"] { + assert!(Name::parse(s).is_err(), "{s}"); + } + } + + #[test] + fn serialised_as_text() { + let m = Month::parse("2024-02").unwrap(); + assert_eq!(serde_json::to_string(&m).unwrap(), "\"2024-02\""); + let back: Month = serde_json::from_str("\"2024-02\"").unwrap(); + assert_eq!(back, m); + } +} diff --git a/rust/morca-sdk/src/pipeline.rs b/rust/morca-sdk/src/pipeline.rs new file mode 100644 index 00000000..d4201ca8 --- /dev/null +++ b/rust/morca-sdk/src/pipeline.rs @@ -0,0 +1,817 @@ +//! Defining a pipeline: its name, steps in order, partitions, schedule and +//! retries, built at start-up and checked when built. +//! +//! The builder is typed: each step takes what the step before it makes, so +//! a step whose input doesn't fit the step before it doesn't compile. A +//! first `mapped` step takes the partition key, a `gather` after a mapped +//! step takes a `Vec` of its outputs in key order, and a `gather` after a +//! gather takes its output. Step names, datasets and settings are checked +//! by [`Pipeline::build`], which a binary calls before any command. +//! +//! Across builds, an edge is checked at run time: a step's output is kept +//! as `{"type": , "value": }`, and a resume decodes +//! the outputs an earlier build stored only into the same type name. + +use std::any::type_name; +use std::marker::PhantomData; +use std::sync::Arc; + +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; + +use crate::ctx::Ctx; +use crate::partition::Partition; + +/// A step's failure, from `?` on any error or from [`fail`], or a request +/// to try later, from [`defer`]. +#[derive(Debug)] +pub enum StepError { + /// The attempt failed with this message; the step's retries try again. + Failed(String), + /// The step can't do its work yet: the attempt ends `deferred` and the + /// run starts again in `seconds`, spending no retries. + Defer { seconds: u64, why: String }, +} + +impl From for StepError { + fn from(e: E) -> Self { + StepError::Failed(e.to_string()) + } +} + +impl StepError { + pub(crate) fn message(&self) -> String { + match self { + StepError::Failed(m) => m.clone(), + StepError::Defer { seconds, why } => format!("defer {seconds}: {why}"), + } + } +} + +/// What a step returns. +pub type StepResult = Result; + +/// Fails the attempt with `msg`. +pub fn fail(msg: impl Into) -> StepError { + StepError::Failed(msg.into()) +} + +/// Ends the attempt `deferred`: the run starts again in `seconds` (at least +/// 1), and neither the step's retries nor the run's are spent. +pub fn defer(seconds: u64, why: impl Into) -> StepError { + StepError::Defer { + seconds: seconds.max(1), + why: why.into(), + } +} + +/// A step's name and settings. A `&str` converts to one with the defaults. +#[derive(Clone, Debug, Default)] +pub struct StepSpec { + pub(crate) name: String, + pub(crate) retries: u32, + pub(crate) backoff_seconds: u64, + pub(crate) timeout_seconds: u64, + pub(crate) version: u64, + pub(crate) reads: Vec, + pub(crate) writes: Vec, +} + +/// A step named `name`, to set more on. +pub fn step(name: &str) -> StepSpec { + StepSpec { + name: name.to_string(), + ..Default::default() + } +} + +impl From<&str> for StepSpec { + fn from(name: &str) -> Self { + step(name) + } +} + +impl StepSpec { + /// Tries again up to `n` times in the same run when an attempt fails. + pub fn retries(mut self, n: u32) -> Self { + self.retries = n; + self + } + /// Waits this long before the first retry, doubling each time. + pub fn backoff_seconds(mut self, s: u64) -> Self { + self.backoff_seconds = s; + self + } + /// Stops the process if an attempt is still running after `s` seconds + /// (the heartbeat's watchdog), failing the attempt and the run. + pub fn timeout_seconds(mut self, s: u64) -> Self { + self.timeout_seconds = s; + self + } + /// The step's data version: a new one makes keys done at another + /// version not done for a fill. + pub fn version(mut self, v: u64) -> Self { + self.version = v; + self + } + /// The warehouse datasets (`ns.table`) the step reads. + pub fn reads, S: Into>(mut self, ds: I) -> Self { + self.reads = ds.into_iter().map(Into::into).collect(); + self + } + /// The warehouse datasets (`ns.table`) the step commits to. + pub fn writes, S: Into>(mut self, ds: I) -> Self { + self.writes = ds.into_iter().map(Into::into).collect(); + self + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + Mapped, + Gather, +} + +impl Kind { + pub(crate) fn as_str(self) -> &'static str { + match self { + Kind::Mapped => "mapped", + Kind::Gather => "gather", + } + } +} + +type RunFn = dyn Fn(&mut Ctx, &[u8]) -> StepResult> + Send + Sync; + +pub(crate) struct StepDef { + pub(crate) spec: StepSpec, + pub(crate) kind: Kind, + pub(crate) after: Option, + pub(crate) function: String, + pub(crate) input: String, + pub(crate) output: String, + pub(crate) run: Arc, +} + +/// A value as a step's output or input row keeps it. +pub(crate) fn encode(v: &T) -> Result, String> { + serde_json::to_vec(&json!({"type": type_name::(), "value": v})) + .map_err(|e| format!("morca: can't encode a {}: {e}", type_name::())) +} + +/// The `value` of an encoded `type`; an error if it was encoded as another. +pub(crate) fn decode_value(bytes: &[u8], want: &str) -> Result { + let v: Value = serde_json::from_slice(bytes) + .map_err(|e| format!("morca: a stored value isn't morca-sdk's encoding: {e}"))?; + let got = v.get("type").and_then(Value::as_str).unwrap_or(""); + if got != want { + return Err(format!( + "morca: a stored value is a {}, the step takes a {want}", + if got.is_empty() { + "value of another encoding" + } else { + got + } + )); + } + Ok(v.get("value").cloned().unwrap_or(Value::Null)) +} + +fn decode(bytes: &[u8]) -> StepResult { + let v = decode_value(bytes, type_name::()).map_err(StepError::Failed)?; + serde_json::from_value(v) + .map_err(|e| fail(format!("morca: can't decode a {}: {e}", type_name::()))) +} + +fn wrap(f: F) -> Arc +where + A: DeserializeOwned + 'static, + R: Serialize + 'static, + F: Fn(&mut Ctx, A) -> StepResult + Send + Sync + 'static, +{ + Arc::new(move |ctx: &mut Ctx, input: &[u8]| { + let a: A = decode(input)?; + let r = f(ctx, a)?; + encode(&r).map_err(StepError::Failed) + }) +} + +/// A cron schedule: five fields, in a zone (UTC unless set). +#[derive(Clone, Debug)] +pub struct Cron { + pub(crate) expr: String, + pub(crate) tz: String, + pub(crate) fills: String, + pub(crate) catchup: bool, +} + +impl Cron { + /// Fires at `expr` (`minute hour day month weekday`), in UTC, each fire + /// filling the partition before it, catching up on missed fires. + pub fn new(expr: &str) -> Self { + Cron { + expr: expr.to_string(), + tz: "UTC".into(), + fills: "previous".into(), + catchup: true, + } + } + /// The IANA zone the expression is in. + pub fn tz(mut self, tz: &str) -> Self { + self.tz = tz.to_string(); + self + } + /// Each fire fills the partition it's in, not the one before. + pub fn fills_current(mut self) -> Self { + self.fills = "current".into(); + self + } + /// Missed fires are skipped instead of run late. + pub fn no_catchup(mut self) -> Self { + self.catchup = false; + self + } +} + +/// describe's `source`: the repository and commit a binary was built from. +#[derive(Clone, Debug, Default)] +pub struct Source { + pub repo: String, + pub commit: String, + pub dirty: bool, +} + +/// The source from `MORCA_SOURCE`, `MORCA_COMMIT` and `MORCA_DIRTY` in the +/// build's environment (`cargo build`), as `mojo build -D` sets them for a +/// Mojo pipeline. +#[macro_export] +macro_rules! source_from_build_env { + () => { + $crate::Source { + repo: option_env!("MORCA_SOURCE").unwrap_or("").to_string(), + commit: option_env!("MORCA_COMMIT").unwrap_or("").to_string(), + dirty: matches!(option_env!("MORCA_DIRTY"), Some("1") | Some("true")), + } + }; +} + +#[derive(Clone, Debug)] +pub(crate) struct Config { + pub(crate) name: String, + pub(crate) kind: &'static str, + pub(crate) start: Option, + pub(crate) keys_dataset: Option<(String, String)>, + pub(crate) schedule: Option, + pub(crate) max_runs: u32, + pub(crate) run_retries: u32, + pub(crate) retry_seconds: u64, + pub(crate) max_retry_seconds: u64, + pub(crate) sensor: bool, + pub(crate) keys_per_run: u32, + pub(crate) source: Source, +} + +/// Typestate: no step yet. +pub struct Start; +/// Typestate: the last step is mapped. +pub struct AfterMapped; +/// Typestate: the last step is a gather. +pub struct AfterGather; + +/// A pipeline being defined, over keys `K`, whose last step makes `Out`. +pub struct Pipeline { + cfg: Config, + steps: Vec, + key_check: fn(&str) -> Result, + _p: Marker, +} + +type Marker = PhantomData (K, Out, S)>; + +fn key_check(s: &str) -> Result { + K::parse(s).map(|k| k.to_string()) +} + +fn key_input(s: &str) -> Result, String> { + encode(&K::parse(s)?) +} + +fn keys_input(keys: &[String]) -> Result, String> { + let ks: Result, String> = keys.iter().map(|k| K::parse(k)).collect(); + encode(&ks?) +} + +impl Pipeline { + /// A pipeline named `name` over keys of type `K`. + pub fn new(name: &str) -> Self { + Pipeline { + cfg: Config { + name: name.to_string(), + kind: K::KIND, + start: None, + keys_dataset: None, + schedule: None, + max_runs: 1, + run_retries: 3, + retry_seconds: 0, + max_retry_seconds: 0, + sensor: false, + keys_per_run: 1, + source: Source::default(), + }, + steps: vec![], + key_check: key_check::, + _p: PhantomData, + } + } + + /// The first step, once per key: takes the key. + pub fn mapped(self, spec: impl Into, f: F) -> Pipeline + where + U: Serialize + DeserializeOwned + 'static, + F: Fn(&mut Ctx, K) -> StepResult + Send + Sync + 'static, + { + self.push::(spec.into(), Kind::Mapped, f) + } + + /// The first step, once per run: takes the run's keys. + pub fn gather(self, spec: impl Into, f: F) -> Pipeline + where + U: Serialize + DeserializeOwned + 'static, + F: Fn(&mut Ctx, Vec) -> StepResult + Send + Sync + 'static, + { + self.push::, U, F, AfterGather>(spec.into(), Kind::Gather, f) + } +} + +impl Pipeline { + /// A step once per key, after a mapped step: takes its output. + pub fn mapped(self, spec: impl Into, f: F) -> Pipeline + where + U: Serialize + DeserializeOwned + 'static, + F: Fn(&mut Ctx, Out) -> StepResult + Send + Sync + 'static, + { + self.push::(spec.into(), Kind::Mapped, f) + } + + /// A step once per run, after a mapped step: takes its outputs, in key + /// order. + pub fn gather(self, spec: impl Into, f: F) -> Pipeline + where + U: Serialize + DeserializeOwned + 'static, + F: Fn(&mut Ctx, Vec) -> StepResult + Send + Sync + 'static, + { + self.push::, U, F, AfterGather>(spec.into(), Kind::Gather, f) + } +} + +impl Pipeline { + /// A step once per run, after a gather: takes its output. (A mapped + /// step can't follow a gather.) + pub fn gather(self, spec: impl Into, f: F) -> Pipeline + where + U: Serialize + DeserializeOwned + 'static, + F: Fn(&mut Ctx, Out) -> StepResult + Send + Sync + 'static, + { + self.push::(spec.into(), Kind::Gather, f) + } +} + +impl Pipeline { + fn push(self, spec: StepSpec, kind: Kind, f: F) -> Pipeline + where + A: DeserializeOwned + 'static, + U: Serialize + DeserializeOwned + 'static, + F: Fn(&mut Ctx, A) -> StepResult + Send + Sync + 'static, + { + let mut steps = self.steps; + let after = steps.last().map(|s| s.spec.name.clone()); + steps.push(StepDef { + spec, + kind, + after, + function: type_name::().to_string(), + input: type_name::().to_string(), + output: type_name::().to_string(), + run: wrap::(f), + }); + Pipeline { + cfg: self.cfg, + steps, + key_check: self.key_check, + _p: PhantomData, + } + } + + /// The first key a fill covers, such as `2024-01`. + pub fn start(mut self, key: &str) -> Self { + self.cfg.start = Some(key.to_string()); + self + } + /// Runs on a cron schedule. + pub fn schedule(mut self, cron: Cron) -> Self { + self.cfg.schedule = Some(cron); + self + } + /// At most this many runs at once (default 1). + pub fn max_runs(mut self, n: u32) -> Self { + self.cfg.max_runs = n; + self + } + /// Starts of a run after a failure (default 3). + pub fn run_retries(mut self, n: u32) -> Self { + self.cfg.run_retries = n; + self + } + /// The backoff between a run's retries, doubling up to `max` seconds. + pub fn retry_seconds(mut self, first: u64, max: u64) -> Self { + self.cfg.retry_seconds = first; + self.cfg.max_retry_seconds = max; + self + } + /// A sensor: the scheduler runs it when a dataset it reads, and no step + /// writes, gets a snapshot another program committed. + pub fn sensor(mut self) -> Self { + self.cfg.sensor = true; + self + } + /// How many keys a queued run covers (default 1). + pub fn keys_per_run(mut self, n: u32) -> Self { + self.cfg.keys_per_run = n; + self + } + /// A keyed pipeline's keys: the distinct values of `column` in the + /// warehouse dataset `dataset`. + pub fn keys_from(mut self, dataset: &str, column: &str) -> Self { + self.cfg.keys_dataset = Some((dataset.to_string(), column.to_string())); + self + } + /// describe's `source` (see [`source_from_build_env!`]). + pub fn source(mut self, source: Source) -> Self { + self.cfg.source = source; + self + } + + /// Checks the definition and makes the pipeline a binary runs. + pub fn build(self) -> Result + where + K: Partition, + { + let built = Built { + cfg: self.cfg, + steps: self.steps, + key_check: self.key_check, + key_input: key_input::, + keys_input: keys_input::, + }; + built.check()?; + Ok(built) + } +} + +/// A pipeline, defined and checked: hand it to [`crate::main`]. +pub struct Built { + pub(crate) cfg: Config, + pub(crate) steps: Vec, + pub(crate) key_check: fn(&str) -> Result, + pub(crate) key_input: fn(&str) -> Result, String>, + pub(crate) keys_input: fn(&[String]) -> Result, String>, +} + +fn plain_name(s: &str) -> bool { + !s.is_empty() + && s.len() <= 64 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') +} + +/// `namespace.table`: a dot with a name either side, and nothing that would +/// leave the warehouse. +pub(crate) fn is_dataset(name: &str) -> bool { + match name.rfind('.') { + Some(dot) => { + dot > 0 + && dot < name.len() - 1 + && !name.contains('/') + && !name.contains("..") + && !name.contains(char::is_whitespace) + } + None => false, + } +} + +impl Built { + fn check(&self) -> Result<(), String> { + let c = &self.cfg; + let bad = |m: String| Err(format!("morca: pipeline {}: {m}", c.name)); + if !plain_name(&c.name) { + return bad("a pipeline's name is 1 to 64 of A-Z a-z 0-9 _ -".into()); + } + if self.steps.is_empty() { + return bad("it has no steps".into()); + } + let mut names: Vec<&str> = vec![]; + for s in &self.steps { + let n = s.spec.name.as_str(); + if !plain_name(n) { + return bad(format!( + "step '{n}': a step's name is 1 to 64 of A-Z a-z 0-9 _ -" + )); + } + if names.contains(&n) { + return bad(format!("two steps are named {n}")); + } + names.push(n); + for d in s.spec.reads.iter().chain(&s.spec.writes) { + if !is_dataset(d) { + return bad(format!("step {n}: '{d}' isn't a dataset (namespace.table)")); + } + } + if s.spec.writes.iter().any(|d| s.spec.reads.contains(d)) { + return bad(format!("step {n} reads a dataset it writes")); + } + if !cfg!(feature = "tables") && !(s.spec.reads.is_empty() && s.spec.writes.is_empty()) { + return bad(format!( + "step {n} declares datasets: build morca-sdk with the `tables` feature" + )); + } + } + if let Some(k) = &c.start { + (self.key_check)(k).map_err(|e| format!("morca: pipeline {}: start: {e}", c.name))?; + if c.kind == "keyed" { + return bad("a keyed pipeline has no start".into()); + } + } + match (&c.keys_dataset, c.kind == "keyed") { + (None, true) => return bad("a keyed pipeline needs keys_from(dataset, column)".into()), + (Some(_), false) => { + return bad("keys_from is for a keyed pipeline (keys of Name)".into()); + } + (Some((d, col)), true) => { + if !is_dataset(d) || col.is_empty() { + return bad(format!("keys_from({d}, {col}): not a dataset and a column")); + } + if !cfg!(feature = "tables") { + return bad("a keyed pipeline needs the `tables` feature".into()); + } + } + _ => {} + } + if let Some(s) = &c.schedule + && s.expr.split_whitespace().count() != 5 + { + return bad(format!( + "schedule '{}': a cron expression has five fields", + s.expr + )); + } + if c.sensor && self.sensed().is_empty() { + return bad("a sensor reads a dataset no step writes".into()); + } + if c.max_runs < 1 || c.keys_per_run < 1 { + return bad("max_runs and keys_per_run are at least 1".into()); + } + Ok(()) + } + + pub(crate) fn name(&self) -> &str { + &self.cfg.name + } + + /// Each dataset a step reads and no step writes, once, in step order, + /// then a keyed pipeline's keys dataset. + pub(crate) fn sensed(&self) -> Vec { + let written: Vec<&String> = self.steps.iter().flat_map(|s| &s.spec.writes).collect(); + let mut out: Vec = vec![]; + for s in &self.steps { + for d in &s.spec.reads { + if !written.contains(&d) && !out.contains(d) { + out.push(d.clone()); + } + } + } + if let Some((d, _)) = &self.cfg.keys_dataset + && !out.contains(d) + { + out.push(d.clone()); + } + out + } + + /// The registry hash: FNV-1a (64 bits) over the pipeline's name, its + /// steps' names, kinds, edges, functions, datasets and versions, and its + /// partitions, as the Mojo binary's text (src/morca/pipeline.mojo), + /// printed `0x` and lower-case hex. + pub fn registry_hash(&self) -> String { + let mut all = format!("{}\n", self.cfg.name); + for s in &self.steps { + all.push_str(&format!( + "{}\t{}\t{}\t{}", + s.spec.name, + s.kind.as_str(), + s.after.as_deref().unwrap_or(""), + s.function + )); + for d in &s.spec.reads { + all.push_str(&format!("\treads {d}")); + } + for d in &s.spec.writes { + all.push_str(&format!("\twrites {d}")); + } + if s.spec.version > 0 { + all.push_str(&format!("\tversion {}", s.spec.version)); + } + all.push('\n'); + } + all.push_str(&format!("partitions\t{}\n", self.cfg.kind)); + if let Some((d, c)) = &self.cfg.keys_dataset { + all.push_str(&format!("keys\t{d}\t{c}\n")); + } + format!("0x{:x}", fnv1a64(all.as_bytes())) + } + + /// `step=seconds` for each step with a timeout, comma-separated. + pub(crate) fn timeouts(&self) -> String { + self.steps + .iter() + .filter(|s| s.spec.timeout_seconds > 0) + .map(|s| format!("{}={}", s.spec.name, s.spec.timeout_seconds)) + .collect::>() + .join(",") + } + + /// The DAG as one line of JSON (docs/pipeline-protocol.md, "describe"): + /// `pipeline` first, `protocol` 1, and no `morca` key, since the binary + /// isn't built with morca-mojo. + pub fn describe(&self) -> String { + let c = &self.cfg; + let mut parts = vec![("kind", J::v(c.kind)), ("start", J::v(&c.start))]; + if let Some((d, col)) = &c.keys_dataset { + parts.push(("dataset", J::v(d))); + parts.push(("column", J::v(col))); + } + let schedule = match &c.schedule { + Some(s) => { + J::V(json!({"cron": s.expr, "tz": s.tz, "fills": s.fills, "catchup": s.catchup})) + } + None => J::V(Value::Null), + }; + let steps: Vec = self + .steps + .iter() + .map(|s| { + let mut o = vec![ + ("name", J::v(&s.spec.name)), + ("kind", J::v(s.kind.as_str())), + ("after", J::v(&s.after)), + ("function", J::v(&s.function)), + ("input", J::v(&s.input)), + ("output", J::v(&s.output)), + ]; + if s.spec.timeout_seconds > 0 { + o.push(("timeout_seconds", J::v(s.spec.timeout_seconds))); + } + if s.spec.version > 0 { + o.push(("version", J::v(s.spec.version))); + } + o.push(("reads", J::v(&s.spec.reads))); + o.push(("writes", J::v(&s.spec.writes))); + J::Obj(o) + }) + .collect(); + let mut top = vec![ + ("pipeline", J::v(&c.name)), + ("registry_hash", J::v(self.registry_hash())), + ("partitions", J::Obj(parts)), + ("schedule", schedule), + ("max_runs", J::v(c.max_runs)), + ("run_retries", J::v(c.run_retries)), + ]; + if c.retry_seconds > 0 { + top.push(("retry_seconds", J::v(c.retry_seconds))); + } + if c.max_retry_seconds > 0 { + top.push(("max_retry_seconds", J::v(c.max_retry_seconds))); + } + top.extend([ + ("sensor", J::v(c.sensor)), + ("keys_per_run", J::v(c.keys_per_run)), + ("workers", J::v(2)), + ("steps", J::Arr(steps)), + ("tasks", J::V(json!([]))), + ( + "source", + J::V(json!({"repo": c.source.repo, "commit": c.source.commit, "dirty": c.source.dirty})), + ), + ("protocol", J::v(crate::PROTOCOL_VERSION)), + ]); + J::Obj(top).text() + } +} + +/// JSON whose objects keep their keys in order, so describe starts with +/// `pipeline` (serde_json's own map sorts its keys). +enum J { + V(Value), + Obj(Vec<(&'static str, J)>), + Arr(Vec), +} + +impl J { + fn v(x: T) -> J { + J::V(serde_json::to_value(x).unwrap_or(Value::Null)) + } + fn text(&self) -> String { + match self { + J::V(v) => v.to_string(), + J::Arr(a) => format!("[{}]", a.iter().map(J::text).collect::>().join(",")), + J::Obj(o) => format!( + "{{{}}}", + o.iter() + .map(|(k, v)| format!("{}:{}", Value::String(k.to_string()), v.text())) + .collect::>() + .join(",") + ), + } + } +} + +/// The 64-bit FNV-1a hash. +pub(crate) fn fnv1a64(b: &[u8]) -> u64 { + let mut h: u64 = 0xcbf29ce484222325; + for &x in b { + h ^= x as u64; + h = h.wrapping_mul(0x100000001b3); + } + h +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::partition::Month; + + fn days(_: &mut Ctx, m: Month) -> StepResult { + Ok(m.days()) + } + fn total(_: &mut Ctx, ds: Vec) -> StepResult { + Ok(ds.iter().map(|d| *d as u64).sum()) + } + + fn toy() -> Pipeline { + Pipeline::::new("t") + .start("2024-01") + .mapped("days", days) + .gather(step("total").timeout_seconds(5), total) + } + + #[test] + fn describe_and_hash() { + let p = toy().build().unwrap(); + let d: Value = serde_json::from_str(&p.describe()).unwrap(); + assert!(p.describe().starts_with("{\"pipeline\":\"t\",")); + assert_eq!(d["protocol"], 1); + assert!(d.get("morca").is_none()); + assert_eq!(d["steps"][1]["after"], "days"); + assert_eq!(d["steps"][1]["timeout_seconds"], 5); + assert_eq!(d["registry_hash"], p.registry_hash()); + assert_eq!(p.registry_hash(), toy().build().unwrap().registry_hash()); + assert_eq!(p.timeouts(), "total=5"); + // Another DAG, another hash. + let q = Pipeline::::new("t") + .mapped("days", days) + .build() + .unwrap(); + assert_ne!(p.registry_hash(), q.registry_hash()); + } + + #[test] + fn checked_when_built() { + let dup = Pipeline::::new("t") + .mapped("a", days) + .gather("a", total) + .build(); + assert!(dup.err().unwrap().contains("two steps are named a")); + let bad = Pipeline::::new("t") + .start("2024-13") + .mapped("a", days) + .build(); + assert!(bad.is_err()); + let sensor = Pipeline::::new("t") + .sensor() + .mapped("a", days) + .build(); + assert!(sensor.is_err()); + let empty = Pipeline::::new("t").build(); + assert!(empty.is_err()); + } + + #[test] + fn edges_checked_at_run_time() { + let out = encode(&7u32).unwrap(); + assert!(decode_value(&out, type_name::()).is_ok()); + let e = decode_value(&out, type_name::()).unwrap_err(); + assert!(e.contains("is a u32"), "{e}"); + } +} diff --git a/rust/morca-sdk/src/store/mod.rs b/rust/morca-sdk/src/store/mod.rs new file mode 100644 index 00000000..a59d9f3f --- /dev/null +++ b/rust/morca-sdk/src/store/mod.rs @@ -0,0 +1,378 @@ +//! The run store (docs/pipeline-protocol.md, "The run store"): the +//! `RunStore` contract on Postgres (`MORCA_DATABASE_URL=postgres://...`) and +//! on a SQLite file (`sqlite:`), each statement written as morca's own +//! (`src/morca/_internal/db/runs.mojo`, `steps.mojo`, `sqlite_runs.mojo`). +//! +//! [`Owned`] is the store as a run's owner: the calls for the run (attempts, +//! skips, reads and commits, its end) retried for `MORCA_DB_RETRY_SECONDS` +//! on a new connection, except a lost or cancelled run, which is final. + +pub(crate) mod pg; +pub(crate) mod sqlite; + +use std::collections::HashMap; +use std::fmt; +use std::time::{Duration, Instant}; + +/// A store error: `Final` (the run was lost or cancelled, or the commit +/// fence said no) isn't retried; `Retry` (the database) is. +#[derive(Debug, Clone)] +pub(crate) enum DbError { + Final(String), + Retry(String), +} + +impl fmt::Display for DbError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DbError::Final(m) | DbError::Retry(m) => f.write_str(m), + } + } +} + +impl From for DbError { + fn from(e: postgres::Error) -> Self { + DbError::Retry(format!("Postgres: {e}")) + } +} + +impl From for DbError { + fn from(e: rusqlite::Error) -> Self { + DbError::Retry(format!("SQLite: {e}")) + } +} + +pub(crate) type Res = Result; + +pub(crate) const RUNNING: &str = "running"; +pub(crate) const SUCCEEDED: &str = "succeeded"; +pub(crate) const FAILED: &str = "failed"; +pub(crate) const SKIPPED: &str = "skipped"; +pub(crate) const DEFERRED: &str = "deferred"; +pub(crate) const CANCELLED: &str = "cancelled"; +/// An attempt a process left running, when no crash says why. +pub(crate) const INTERRUPTED: &str = "interrupted: the process running it stopped"; + +pub(crate) fn lost(run: i64) -> DbError { + DbError::Final(format!( + "morca: lost run {run} to another process; stopping" + )) +} + +pub(crate) fn cancelled(run: i64) -> DbError { + DbError::Final(format!("morca: run {run} was cancelled; stopping")) +} + +#[derive(Debug, Clone)] +pub(crate) struct RunInfo { + pub pipeline: String, + pub registry_hash: String, + pub partitions: String, + pub status: String, + pub owner: String, +} + +/// The fence's look at a run before a commit. +pub(crate) struct Fence { + pub owner: String, + pub status: String, + pub fresh: bool, +} + +/// An attempt as `start_attempt` made it. +#[derive(Clone, Copy)] +pub(crate) struct Attempt { + /// 1, 2, ... for the step row. + pub number: i64, + /// The `attempts` row. + pub id: i64, +} + +/// One store: a Postgres connection or a SQLite file. +pub(crate) trait RunStore: Send { + /// Postgres: checks the schema has what this binary uses (the scheduler + /// migrates it). SQLite: makes or brings up the file's schema. + fn migrate(&mut self) -> Res<()>; + fn reconnect(&mut self) -> Res<()>; + fn run_info(&mut self, run: i64) -> Res>; + fn claim(&mut self, run: i64, owner: &str, stale: f64) -> Res; + fn version(&mut self, pipeline: &str, hash: &str, describe: &str, image: &str) -> Res; + fn ran_version(&mut self, run: i64, version: i64) -> Res<()>; + /// Fails the run's running attempts and steps with `why`; how many + /// attempts. + fn interrupt(&mut self, run: i64, why: &str) -> Res; + /// The run's succeeded steps: (step, partition) to output. + fn prior(&mut self, run: i64) -> Res>>; + fn run_token(&mut self, run: i64) -> Res; + #[allow(clippy::too_many_arguments)] + fn start_attempt( + &mut self, + run: i64, + owner: &str, + hash: &str, + step: &str, + partition: &str, + ord: i32, + input: &[u8], + ) -> Res; + #[allow(clippy::too_many_arguments)] + fn finish_attempt( + &mut self, + run: i64, + owner: &str, + step: &str, + partition: &str, + attempt: i64, + status: &str, + output: &[u8], + error: &str, + ) -> Res<()>; + fn skip( + &mut self, + run: i64, + owner: &str, + step: &str, + partition: &str, + ord: i32, + reason: &str, + ) -> Res<()>; + fn set_log(&mut self, attempt_id: i64, location: &str) -> Res<()>; + fn pinned_reads(&mut self, run: i64, step: &str, partition: &str) -> Res>; + fn record( + &mut self, + commit: bool, + attempt_id: i64, + table: &str, + snapshot: i64, + dataset: &str, + ) -> Res<()>; + fn fence(&mut self, run: i64, fresh_seconds: f64) -> Res>; + fn cancel_requested(&mut self, run: i64) -> Res; + fn finish_run(&mut self, run: i64, owner: &str, status: &str) -> Res<()>; + fn defer_run(&mut self, run: i64, owner: &str, until: i64) -> Res<()>; + /// The heartbeat's own connection. + fn beater(&self, run: i64, owner: &str, timeouts: &str) -> Box; +} + +/// The store for `MORCA_DATABASE_URL`. +pub(crate) fn open(url: &str) -> Res> { + if let Some(path) = sqlite::path_of(url) { + return Ok(Box::new(sqlite::SqliteStore::open(&path)?)); + } + Ok(Box::new(pg::PgStore::connect(url)?)) +} + +/// `MORCA_STALE_SECONDS` (default 30): the scheduler's stale limit. +pub(crate) fn stale_seconds() -> f64 { + std::env::var("MORCA_STALE_SECONDS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|v| *v > 0.0) + .unwrap_or(30.0) +} + +/// `MORCA_DB_RETRY_SECONDS`, default the stale limit. +fn retry_budget() -> f64 { + std::env::var("MORCA_DB_RETRY_SECONDS") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|v| *v >= 0.0) + .unwrap_or_else(stale_seconds) +} + +/// The store as the owner of one run. +pub(crate) struct Owned { + pub store: Box, + pub run: i64, + pub owner: String, + pub hash: String, + /// The `attempts` row of the attempt last started. + pub attempt_id: i64, +} + +impl Owned { + /// Runs `f`, retrying a database error on a new connection for + /// `budget` seconds: waits 0.25 s, doubling up to 4 s. + fn retry(&mut self, budget: f64, mut f: impl FnMut(&mut dyn RunStore) -> Res) -> Res { + let end = Instant::now() + Duration::from_secs_f64(budget); + let mut wait = 0.25; + loop { + match f(self.store.as_mut()) { + Ok(v) => return Ok(v), + Err(DbError::Final(m)) => return Err(DbError::Final(m)), + Err(DbError::Retry(m)) => { + if Instant::now() + Duration::from_secs_f64(wait) > end { + return Err(DbError::Retry(m)); + } + eprintln!("morca: {m}; retrying in {wait} s"); + std::thread::sleep(Duration::from_secs_f64(wait)); + wait = (wait * 2.0_f64).min(4.0); + let _ = self.store.reconnect(); + } + } + } + } + + pub fn start_attempt( + &mut self, + step: &str, + partition: &str, + ord: i32, + input: &[u8], + ) -> Res { + let (run, owner, hash) = (self.run, self.owner.clone(), self.hash.clone()); + let a = self.retry(retry_budget(), |s| { + s.start_attempt(run, &owner, &hash, step, partition, ord, input) + })?; + self.attempt_id = a.id; + Ok(a.number) + } + + pub fn finish_attempt( + &mut self, + step: &str, + partition: &str, + attempt: i64, + status: &str, + output: &[u8], + error: &str, + ) -> Res<()> { + let (run, owner) = (self.run, self.owner.clone()); + self.retry(retry_budget(), |s| { + s.finish_attempt(run, &owner, step, partition, attempt, status, output, error) + }) + } + + pub fn skip(&mut self, step: &str, partition: &str, ord: i32, reason: &str) -> Res<()> { + let (run, owner) = (self.run, self.owner.clone()); + self.retry(retry_budget(), |s| { + s.skip(run, &owner, step, partition, ord, reason) + }) + } + + pub fn set_log(&mut self, location: &str) -> Res<()> { + let id = self.attempt_id; + self.retry(retry_budget(), |s| s.set_log(id, location)) + } + + pub fn pinned_reads(&mut self, step: &str, partition: &str) -> Res> { + let run = self.run; + self.retry(retry_budget(), |s| s.pinned_reads(run, step, partition)) + } + + pub fn run_token(&mut self) -> Res { + let run = self.run; + self.retry(retry_budget(), |s| s.run_token(run)) + } + + /// Records a read or a commit of the current attempt. + pub fn record(&mut self, commit: bool, table: &str, snapshot: i64, dataset: &str) -> Res<()> { + let id = self.attempt_id; + self.retry(retry_budget(), |s| { + s.record(commit, id, table, snapshot, dataset) + }) + } + + /// Whether this process may still commit: it owns the run, running, + /// with a heartbeat fresher than half the stale limit. A database it + /// can't reach is retried for half the stale limit, then the commit is + /// refused too. + pub fn fence(&mut self) -> Res<()> { + let half = stale_seconds() / 2.0; + let run = self.run; + let got = self + .retry(half, |s| s.fence(run, half)) + .map_err(|e| match e { + DbError::Retry(m) => DbError::Final(format!("morca: not committing: {m}")), + f => f, + })?; + let Some(f) = got else { + return Err(DbError::Final(format!( + "morca: not committing: no run {run}" + ))); + }; + if f.owner != self.owner { + return Err(DbError::Final(format!( + "morca: not committing: run {run} is held by {}", + if f.owner.is_empty() { + "nobody" + } else { + &f.owner + } + ))); + } + if f.status != RUNNING { + return Err(DbError::Final(format!( + "morca: not committing: run {run} is {}", + f.status + ))); + } + if !f.fresh { + return Err(DbError::Final(format!( + "morca: not committing: run {run}'s heartbeat is older than {half} s" + ))); + } + Ok(()) + } + + pub fn cancel_requested(&mut self) -> Res { + let run = self.run; + self.retry(retry_budget(), |s| s.cancel_requested(run)) + } + + pub fn finish_run(&mut self, status: &str) -> Res<()> { + let (run, owner) = (self.run, self.owner.clone()); + self.retry(retry_budget(), |s| s.finish_run(run, &owner, status)) + } + + pub fn defer_run(&mut self, until: i64) -> Res<()> { + let (run, owner) = (self.run, self.owner.clone()); + self.retry(retry_budget(), |s| s.defer_run(run, &owner, until)) + } +} + +/// `:`: unique per live process, and never `scheduler`. +pub(crate) fn process_owner() -> String { + unsafe extern "C" { + fn gethostname(name: *mut std::ffi::c_char, len: usize) -> std::ffi::c_int; + } + let mut buf = [0u8; 256]; + let host = if unsafe { gethostname(buf.as_mut_ptr().cast(), buf.len()) } == 0 { + let n = buf.iter().position(|&b| b == 0).unwrap_or(buf.len()); + String::from_utf8_lossy(&buf[..n]).into_owned() + } else { + String::new() + }; + let host = if host.is_empty() { + "localhost".to_string() + } else { + host + }; + format!("{host}:{}", std::process::id()) +} + +/// `b` as lower-case hex: how the SQLite file keeps a step's bytes. +pub(crate) fn to_hex(b: &[u8]) -> String { + const H: &[u8; 16] = b"0123456789abcdef"; + let mut s = String::with_capacity(b.len() * 2); + for x in b { + s.push(H[(x >> 4) as usize] as char); + s.push(H[(x & 15) as usize] as char); + } + s +} + +pub(crate) fn from_hex(s: &str) -> Res> { + let nib = |c: u8| match c { + b'0'..=b'9' => Ok(c - b'0'), + b'a'..=b'f' => Ok(c - b'a' + 10), + b'A'..=b'F' => Ok(c - b'A' + 10), + _ => Err(DbError::Retry("morca: not hex in the SQLite store".into())), + }; + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len() / 2); + for i in (0..b.len().saturating_sub(1)).step_by(2) { + out.push(nib(b[i])? << 4 | nib(b[i + 1])?); + } + Ok(out) +} diff --git a/rust/morca-sdk/src/store/pg.rs b/rust/morca-sdk/src/store/pg.rs new file mode 100644 index 00000000..458e1024 --- /dev/null +++ b/rust/morca-sdk/src/store/pg.rs @@ -0,0 +1,610 @@ +//! The run store in Postgres (`orca.*`), with the statements of +//! src/morca/_internal/db/runs.mojo and steps.mojo. Every parameter is cast +//! in the SQL, so the Rust types bound are the ones named here whatever the +//! column's type. + +use std::collections::HashMap; + +use postgres::{Client, NoTls, Transaction}; + +use super::{Attempt, DbError, Fence, INTERRUPTED, Res, RunInfo, RunStore, cancelled, lost}; +use crate::heartbeat::{self, Beater}; + +pub(crate) struct PgStore { + url: String, + client: Client, +} + +fn connect(url: &str) -> Res { + Client::connect(url, NoTls).map_err(|e| DbError::Retry(format!("Postgres: can't connect: {e}"))) +} + +impl PgStore { + pub fn connect(url: &str) -> Res { + Ok(PgStore { + url: url.to_string(), + client: connect(url)?, + }) + } +} + +/// The columns this binary reads or writes, by table: the schema must have +/// them all (the scheduler migrates it; a pipeline binary doesn't). +const COLUMNS: &[(&str, &[&str])] = &[ + ( + "pipelines", + &[ + "name", + "registry_hash", + "describe", + "binary_path", + "version", + ], + ), + ( + "pipeline_versions", + &[ + "id", + "pipeline", + "registry_hash", + "source", + "commit_sha", + "dirty", + "image", + "data_version", + "last_seen", + ], + ), + ( + "runs", + &[ + "id", + "pipeline", + "registry_hash", + "partitions", + "status", + "owner", + "heartbeat", + "ended", + "error", + "version", + "retry_at", + "deferrals", + "cancel_requested", + "cancel_at", + "token", + "instance", + ], + ), + ( + "steps", + &[ + "id", + "run", + "step", + "partition", + "ord", + "status", + "input", + "output", + "error", + ], + ), + ( + "attempts", + &[ + "id", + "step", + "attempt", + "status", + "host", + "registry_hash", + "started", + "ended", + "error", + "log", + ], + ), + ( + "reads", + &["attempt", "table_name", "snapshot_id", "dataset"], + ), + ( + "commits", + &["attempt", "table_name", "snapshot_id", "dataset"], + ), + ("audit", &["username", "action", "run", "detail"]), + ( + "schedulers", + &["id", "ended", "draining_since", "drain_asked", "running"], + ), +]; + +/// Ownership at every step: refreshes the heartbeat if `owner` still holds +/// the run running; None if it doesn't, else whether a cancel was asked. +fn still_owner(tx: &mut Transaction, run: i64, owner: &str) -> Res> { + let rows = tx.query( + "UPDATE orca.runs SET heartbeat = now() WHERE id = $1::bigint AND owner = $2::text \ + AND status = 'running' RETURNING cancel_requested", + &[&run, &owner], + )?; + Ok(rows.first().map(|r| r.get::<_, bool>(0))) +} + +fn fail_running(tx: &mut Transaction, run: i64, why_attempts: &str, why_steps: &str) -> Res { + let n = tx.execute( + "UPDATE orca.attempts a SET status = 'failed', ended = now(), error = $2::text \ + FROM orca.steps s WHERE a.step = s.id AND s.run = $1::bigint AND a.status = 'running'", + &[&run, &why_attempts], + )?; + tx.execute( + "UPDATE orca.steps SET status = 'failed', error = $2::text WHERE run = $1::bigint \ + AND status = 'running'", + &[&run, &why_steps], + )?; + Ok(n) +} + +/// `data_version_sql` (src/morca/_internal/migrations.mojo): the steps' +/// `version=` as `=`, by step name. +const DATA_VERSION: &str = "coalesce((SELECT string_agg((x->>'name') || '=' || (x->>'version'), \ +',' ORDER BY x->>'name') FROM jsonb_array_elements((d)->'steps') AS x WHERE x->>'version' IS NOT \ +NULL), '')"; + +impl RunStore for PgStore { + fn migrate(&mut self) -> Res<()> { + let rows = self.client.query( + "SELECT table_name::text, column_name::text FROM information_schema.columns \ + WHERE table_schema = 'orca'", + &[], + )?; + let have: std::collections::HashSet<(String, String)> = + rows.iter().map(|r| (r.get(0), r.get(1))).collect(); + if !have.iter().any(|(t, _)| t == "runs") { + return Err(DbError::Final( + "morca: the database has no morca schema: start the scheduler or run \ + `morca register` first" + .into(), + )); + } + let mut missing = vec![]; + for (t, cols) in COLUMNS { + for c in *cols { + if !have.contains(&(t.to_string(), c.to_string())) { + missing.push(format!("orca.{t}.{c}")); + } + } + } + if !missing.is_empty() { + return Err(DbError::Final(format!( + "morca: the database's schema is older than this binary (no {}): upgrade the scheduler", + missing.join(", ") + ))); + } + Ok(()) + } + + fn reconnect(&mut self) -> Res<()> { + self.client = connect(&self.url)?; + Ok(()) + } + + fn run_info(&mut self, run: i64) -> Res> { + let rows = self.client.query( + "SELECT pipeline, registry_hash, partitions, status, coalesce(owner, '') \ + FROM orca.runs WHERE id = $1::bigint", + &[&run], + )?; + Ok(rows.first().map(|r| RunInfo { + pipeline: r.get(0), + registry_hash: r.get(1), + partitions: r.get(2), + status: r.get(3), + owner: r.get(4), + })) + } + + fn claim(&mut self, run: i64, owner: &str, stale: f64) -> Res { + let rows = self.client.query( + "UPDATE orca.runs SET status = 'running', owner = $2::text, heartbeat = now(), \ + ended = NULL, error = NULL WHERE id = $1::bigint AND status IN ('queued', \ + 'running', 'failed', 'deferred') AND (heartbeat IS NULL OR heartbeat < now() - \ + make_interval(secs => $3::float8)) RETURNING id", + &[&run, &owner, &stale], + )?; + Ok(rows.len() == 1) + } + + fn version(&mut self, pipeline: &str, hash: &str, describe: &str, image: &str) -> Res { + let sql = format!( + "INSERT INTO orca.pipeline_versions (pipeline, registry_hash, source, commit_sha, \ + dirty, image, data_version) SELECT $1::text, $2::text, coalesce(d->'source'->>'repo', \ + ''), coalesce(d->'source'->>'commit', ''), coalesce((d->'source'->>'dirty')::boolean, \ + false), $4::text, {DATA_VERSION} FROM (SELECT $3::text::jsonb AS d) AS x ON CONFLICT \ + (pipeline, registry_hash, source, commit_sha, dirty, image) DO UPDATE SET last_seen = \ + now(), data_version = excluded.data_version RETURNING id::bigint" + ); + let row = self + .client + .query_one(&sql, &[&pipeline, &hash, &describe, &image])?; + Ok(row.get(0)) + } + + fn ran_version(&mut self, run: i64, version: i64) -> Res<()> { + self.client.execute( + "UPDATE orca.runs SET version = $2::bigint WHERE id = $1::bigint", + &[&run, &version], + )?; + Ok(()) + } + + fn interrupt(&mut self, run: i64, why: &str) -> Res { + let mut tx = self.client.transaction()?; + let n = fail_running( + &mut tx, + run, + if why.is_empty() { INTERRUPTED } else { why }, + if why.is_empty() { "interrupted" } else { why }, + )?; + tx.commit()?; + Ok(n) + } + + fn prior(&mut self, run: i64) -> Res>> { + let rows = self.client.query( + "SELECT step, partition, coalesce(output, ''::bytea) FROM orca.steps WHERE run = \ + $1::bigint AND status = 'succeeded'", + &[&run], + )?; + Ok(rows + .iter() + .map(|r| ((r.get(0), r.get(1)), r.get::<_, Vec>(2))) + .collect()) + } + + fn run_token(&mut self, run: i64) -> Res { + let rows = self.client.query( + "SELECT coalesce(token::text, '') FROM orca.runs WHERE id = $1::bigint", + &[&run], + )?; + Ok(rows.first().map(|r| r.get(0)).unwrap_or_default()) + } + + fn start_attempt( + &mut self, + run: i64, + owner: &str, + hash: &str, + step: &str, + partition: &str, + ord: i32, + input: &[u8], + ) -> Res { + let mut tx = self.client.transaction()?; + match still_owner(&mut tx, run, owner)? { + None => return Err(lost(run)), + Some(true) => return Err(cancelled(run)), + Some(false) => {} + } + let step_id: i64 = tx + .query_one( + "INSERT INTO orca.steps (run, step, partition, ord, status, input) VALUES \ + ($1::bigint, $2::text, $3::text, $4::int, 'running', $5::bytea) ON CONFLICT \ + (run, step, partition) DO UPDATE SET status = 'running', input = \ + EXCLUDED.input, error = NULL RETURNING id::bigint", + &[&run, &step, &partition, &ord, &input], + )? + .get(0); + let number: i64 = tx + .query_one( + "SELECT (coalesce(max(attempt), 0) + 1)::bigint FROM orca.attempts WHERE step = \ + $1::bigint", + &[&step_id], + )? + .get(0); + let id: i64 = tx + .query_one( + "INSERT INTO orca.attempts (step, attempt, status, host, registry_hash) VALUES \ + ($1::bigint, $2::bigint, 'running', $3::text, $4::text) RETURNING id::bigint", + &[&step_id, &number, &owner, &hash], + )? + .get(0); + tx.commit()?; + Ok(Attempt { number, id }) + } + + fn finish_attempt( + &mut self, + run: i64, + owner: &str, + step: &str, + partition: &str, + attempt: i64, + status: &str, + output: &[u8], + error: &str, + ) -> Res<()> { + let mut tx = self.client.transaction()?; + if still_owner(&mut tx, run, owner)?.is_none() { + return Err(lost(run)); + } + let step_id: i64 = tx + .query_one( + "SELECT id::bigint FROM orca.steps WHERE run = $1::bigint AND step = $2::text AND \ + partition = $3::text", + &[&run, &step, &partition], + )? + .get(0); + let err: Option<&str> = if error.is_empty() { None } else { Some(error) }; + tx.execute( + "UPDATE orca.attempts SET status = $3::text, ended = now(), error = $4::text WHERE \ + step = $1::bigint AND attempt = $2::bigint", + &[&step_id, &attempt, &status, &err], + )?; + let out: Option<&[u8]> = if status == super::SUCCEEDED { + Some(output) + } else { + None + }; + tx.execute( + "UPDATE orca.steps SET status = $2::text, output = $3::bytea, error = $4::text WHERE \ + id = $1::bigint", + &[&step_id, &status, &out, &err], + )?; + tx.commit()?; + Ok(()) + } + + fn skip( + &mut self, + run: i64, + owner: &str, + step: &str, + partition: &str, + ord: i32, + reason: &str, + ) -> Res<()> { + let mut tx = self.client.transaction()?; + match still_owner(&mut tx, run, owner)? { + None => return Err(lost(run)), + Some(true) => return Err(cancelled(run)), + Some(false) => {} + } + tx.execute( + "INSERT INTO orca.steps (run, step, partition, ord, status, error) VALUES \ + ($1::bigint, $2::text, $3::text, $4::int, 'skipped', $5::text) ON CONFLICT (run, \ + step, partition) DO UPDATE SET status = 'skipped', error = EXCLUDED.error, output = \ + NULL", + &[&run, &step, &partition, &ord, &reason], + )?; + tx.commit()?; + Ok(()) + } + + fn set_log(&mut self, attempt_id: i64, location: &str) -> Res<()> { + self.client.execute( + "UPDATE orca.attempts SET log = $2::text WHERE id = $1::bigint", + &[&attempt_id, &location], + )?; + Ok(()) + } + + fn pinned_reads(&mut self, run: i64, step: &str, partition: &str) -> Res> { + let rows = self.client.query( + "SELECT r.table_name, r.snapshot_id::bigint FROM orca.reads r JOIN orca.attempts a ON \ + a.id = r.attempt JOIN orca.steps s ON s.id = a.step WHERE s.run = $1::bigint AND \ + s.step = $2::text AND s.partition = $3::text ORDER BY a.attempt DESC", + &[&run, &step, &partition], + )?; + // Newest first, so the first attempt's snapshot is the one left. + let mut out = HashMap::new(); + for r in rows { + out.insert(r.get::<_, String>(0), r.get::<_, i64>(1)); + } + Ok(out) + } + + fn record( + &mut self, + commit: bool, + attempt_id: i64, + table: &str, + snapshot: i64, + dataset: &str, + ) -> Res<()> { + let sql = format!( + "INSERT INTO orca.{} (attempt, table_name, snapshot_id, dataset) VALUES ($1::bigint, \ + $2::text, $3::bigint, nullif($4::text, ''))", + if commit { "commits" } else { "reads" } + ); + self.client + .execute(&sql, &[&attempt_id, &table, &snapshot, &dataset])?; + Ok(()) + } + + fn fence(&mut self, run: i64, fresh_seconds: f64) -> Res> { + let rows = self.client.query( + "SELECT coalesce(owner, ''), status, coalesce(heartbeat > now() - make_interval(secs \ + => $2::float8), false) FROM orca.runs WHERE id = $1::bigint", + &[&run, &fresh_seconds], + )?; + Ok(rows.first().map(|r| Fence { + owner: r.get(0), + status: r.get(1), + fresh: r.get(2), + })) + } + + fn cancel_requested(&mut self, run: i64) -> Res { + let rows = self.client.query( + "SELECT cancel_requested FROM orca.runs WHERE id = $1::bigint", + &[&run], + )?; + Ok(rows.first().is_some_and(|r| r.get::<_, bool>(0))) + } + + fn finish_run(&mut self, run: i64, owner: &str, status: &str) -> Res<()> { + self.client.execute( + "UPDATE orca.runs SET status = $2::text, ended = now(), heartbeat = NULL WHERE id = \ + $1::bigint AND owner = $3::text", + &[&run, &status, &owner], + )?; + Ok(()) + } + + fn defer_run(&mut self, run: i64, owner: &str, until: i64) -> Res<()> { + self.client.execute( + "UPDATE orca.runs SET status = 'deferred', ended = now(), heartbeat = NULL, retry_at \ + = $3::bigint, deferrals = deferrals + 1 WHERE id = $1::bigint AND owner = $2::text", + &[&run, &owner, &until], + )?; + Ok(()) + } + + fn beater(&self, run: i64, owner: &str, timeouts: &str) -> Box { + Box::new(PgBeat { + url: self.url.clone(), + run, + owner: owner.to_string(), + timeouts: timeouts.to_string(), + client: None, + }) + } +} + +struct PgBeat { + url: String, + run: i64, + owner: String, + timeouts: String, + client: Option, +} + +/// The watchdog's query (`_WATCH_SQL` in src/morca/_internal/db/runs.mojo): +/// a cancel whose time has come, or a running attempt past its step's +/// timeout. +const WATCH_SQL: &str = "SELECT 'cancel' AS why, 0::bigint AS attempt, '' AS step, '' AS \ +partition, 0::float8 AS secs FROM orca.runs WHERE id = $1::bigint AND owner = $2::text AND status \ += 'running' AND cancel_requested AND cancel_at <= now() UNION ALL SELECT 'timeout', a.id::bigint, \ +s.step, s.partition, t.secs FROM orca.attempts a JOIN orca.steps s ON s.id = a.step JOIN \ +orca.runs r ON r.id = s.run JOIN (SELECT split_part(x, '=', 1) AS step, split_part(x, '=', \ +2)::float8 AS secs FROM unnest(string_to_array($3::text, ',')) AS x) AS t ON t.step = s.step \ +WHERE s.run = $1::bigint AND r.owner = $2::text AND r.status = 'running' AND a.status = \ +'running' AND a.started <= now() - make_interval(secs => t.secs) LIMIT 1"; + +/// `outdated_sql` (src/morca/_internal/db/handover.mojo): whether the run +/// is outdated, its version's label and the registered one's. +const OUTDATED_SQL: &str = "SELECT r.status = 'running' AND (coalesce(rv.registry_hash <> \ +pv.registry_hash OR rv.commit_sha <> pv.commit_sha OR rv.dirty <> pv.dirty, false) OR \ +coalesce((s.draining_since IS NOT NULL OR s.drain_asked IS NOT NULL) AND s.ended IS NULL AND \ +r.id::text = ANY(string_to_array(s.running, ',')), false)) AS outdated, coalesce(rv.registry_hash \ +<> pv.registry_hash OR rv.commit_sha <> pv.commit_sha OR rv.dirty <> pv.dirty, false) AS differs \ +FROM orca.runs r JOIN orca.pipelines p ON p.name = r.pipeline LEFT JOIN orca.pipeline_versions rv \ +ON rv.id = r.version LEFT JOIN orca.pipeline_versions pv ON pv.id = p.version LEFT JOIN \ +orca.schedulers s ON s.id = r.instance WHERE r.id = $1::bigint"; + +impl PgBeat { + fn stop_reason(&mut self) -> Res> { + let (run, owner) = (self.run, self.owner.clone()); + let c = self.client.as_mut().unwrap(); + let rows = c.query(WATCH_SQL, &[&run, &owner, &self.timeouts])?; + let Some(r) = rows.first() else { + return Ok(None); + }; + let what: String = r.get(0); + let attempt: i64 = r.get(1); + let step: String = r.get(2); + let partition: String = r.get(3); + let secs: f64 = r.get(4); + let mut tx = c.transaction()?; + let why = if what == "timeout" { + let mut why = heartbeat::timed_out_why(secs); + let n = tx.execute( + "UPDATE orca.attempts SET status = 'failed', ended = now(), error = $2::text WHERE \ + id = $1::bigint AND status = 'running'", + &[&attempt, &why], + )?; + if n != 1 { + return Ok(None); // it ended as we looked + } + tx.execute( + "UPDATE orca.steps SET status = 'failed', error = $2::text WHERE id = (SELECT \ + step FROM orca.attempts WHERE id = $1::bigint)", + &[&attempt, &why], + )?; + let named = if partition.is_empty() { + step + } else { + format!("{step} {partition}") + }; + why = format!("step {named} {why}"); + tx.execute( + "UPDATE orca.runs SET status = 'failed', ended = now(), heartbeat = NULL, error = \ + $3::text WHERE id = $1::bigint AND owner = $2::text", + &[&run, &owner, &why], + )?; + why + } else { + let why = heartbeat::CANCEL_WHY.to_string(); + fail_running(&mut tx, run, &why, &why)?; + tx.execute( + "UPDATE orca.runs SET status = 'cancelled', ended = now(), heartbeat = NULL WHERE \ + id = $1::bigint AND owner = $2::text", + &[&run, &owner], + )?; + why + }; + let detail = format!("{owner}: {why}"); + tx.execute( + "INSERT INTO orca.audit (username, action, run, detail) VALUES ('orca', 'stop', \ + $1::bigint, $2::text)", + &[&run, &detail], + )?; + tx.commit()?; + Ok(Some(why)) + } +} + +impl Beater for PgBeat { + fn beat(&mut self) -> Result, String> { + if self.client.is_none() { + self.client = Some(connect(&self.url).map_err(|e| e.to_string())?); + } + let (run, owner) = (self.run, self.owner.clone()); + self.client + .as_mut() + .unwrap() + .execute( + "UPDATE orca.runs SET heartbeat = now() WHERE id = $1::bigint AND owner = \ + $2::text AND status = 'running'", + &[&run, &owner], + ) + .map_err(|e| e.to_string())?; + if let Some(why) = self.stop_reason().map_err(|e| e.to_string())? { + return Ok(Some(why)); + } + // Whether a newer version leads, for `Ctx::stopping`. Not knowing + // keeps the last answer and never stops the beat. + if let Ok(rows) = self.client.as_mut().unwrap().query(OUTDATED_SQL, &[&run]) + && let Some(r) = rows.first() + { + let o: Option = r.get(0); + let differs: Option = r.get(1); + let why = match (o, differs) { + (Some(true), Some(true)) => { + "the pipeline is registered with newer code".to_string() + } + (Some(true), _) => { + "its scheduler instance is draining: a newer one leads".to_string() + } + _ => String::new(), + }; + heartbeat::set_outdated(o.unwrap_or(false), why); + } + Ok(None) + } + + fn reset(&mut self) { + self.client = None; + } +} diff --git a/rust/morca-sdk/src/store/sqlite.rs b/rust/morca-sdk/src/store/sqlite.rs new file mode 100644 index 00000000..d56f7e7e --- /dev/null +++ b/rust/morca-sdk/src/store/sqlite.rs @@ -0,0 +1,631 @@ +//! The run store in a SQLite file (`MORCA_DATABASE_URL=sqlite:`), +//! with the schema of src/morca/_internal/db/sqlite_schema.mojo and the +//! statements of db/sqlite_runs.mojo: Postgres's tables without the `orca.` +//! prefix, times as Unix seconds, a step's bytes as lower-case hex. WAL mode, +//! one writer at a time (`BEGIN IMMEDIATE`), a second's `busy_timeout`. +//! +//! Unlike Postgres, the binary makes the file's schema itself: `migrate` +//! applies the migrations the file hasn't had, recorded in +//! `schema_version` as the Mojo binary records them, so either binary can +//! use a file the other made. + +use std::collections::HashMap; + +use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params}; + +use super::{ + Attempt, DbError, Fence, INTERRUPTED, Res, RunInfo, RunStore, cancelled, from_hex, lost, to_hex, +}; +use crate::heartbeat::{self, Beater}; + +/// Unix seconds, to the millisecond, in SQL. +const NOW: &str = "((julianday('now') - 2440587.5) * 86400.0)"; + +/// The file of `sqlite:` or `sqlite://`. +pub(crate) fn path_of(url: &str) -> Option { + let rest = url.strip_prefix("sqlite:")?; + Some(rest.strip_prefix("//").unwrap_or(rest).to_string()) +} + +fn open(path: &str) -> Res { + if path.is_empty() { + return Err(DbError::Final("morca: no file in the SQLite URL".into())); + } + let c = Connection::open(path)?; + c.busy_timeout(std::time::Duration::from_millis(1000))?; + c.query_row("PRAGMA journal_mode = WAL", [], |_| Ok(()))?; + c.execute_batch("PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON;")?; + Ok(c) +} + +/// src/morca/_internal/db/sqlite_schema.mojo's migrations; a migration's +/// version is its index + 1. +const MIGRATIONS: &[&[&str]] = &[ + &[ + "CREATE TABLE pipelines ( + name TEXT PRIMARY KEY, + registry_hash TEXT NOT NULL, + describe TEXT NOT NULL, + binary_path TEXT, + version INTEGER, + first_seen REAL NOT NULL, + last_seen REAL NOT NULL)", + "CREATE TABLE pipeline_versions ( + id INTEGER PRIMARY KEY, + pipeline TEXT NOT NULL REFERENCES pipelines (name), + registry_hash TEXT NOT NULL, + source TEXT NOT NULL, + commit_sha TEXT NOT NULL, + dirty INTEGER NOT NULL, + image TEXT NOT NULL, + first_seen REAL NOT NULL, + last_seen REAL NOT NULL, + UNIQUE (pipeline, registry_hash, source, commit_sha, dirty, + image))", + "CREATE TABLE runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pipeline TEXT NOT NULL, + registry_hash TEXT NOT NULL, + partitions TEXT NOT NULL, + trigger TEXT NOT NULL, + status TEXT NOT NULL, + owner TEXT, + heartbeat REAL, + created REAL NOT NULL, + started REAL, + ended REAL, + cancel_requested INTEGER NOT NULL DEFAULT 0, + version INTEGER REFERENCES pipeline_versions (id), + error TEXT)", + "CREATE TABLE steps ( + id INTEGER PRIMARY KEY, + run INTEGER NOT NULL REFERENCES runs (id), + step TEXT NOT NULL, + partition TEXT NOT NULL, + ord INTEGER NOT NULL, + status TEXT NOT NULL, + input TEXT, + output TEXT, + error TEXT, + UNIQUE (run, step, partition))", + "CREATE TABLE attempts ( + id INTEGER PRIMARY KEY, + step INTEGER NOT NULL REFERENCES steps (id), + attempt INTEGER NOT NULL, + status TEXT NOT NULL, + host TEXT NOT NULL, + registry_hash TEXT NOT NULL, + started REAL NOT NULL, + ended REAL, + error TEXT, + log TEXT, + UNIQUE (step, attempt))", + "CREATE TABLE reads ( + attempt INTEGER NOT NULL REFERENCES attempts (id), + table_name TEXT NOT NULL, + snapshot_id INTEGER NOT NULL, + dataset TEXT)", + "CREATE TABLE commits ( + attempt INTEGER NOT NULL REFERENCES attempts (id), + table_name TEXT NOT NULL, + snapshot_id INTEGER NOT NULL, + dataset TEXT)", + "CREATE INDEX reads_attempt ON reads (attempt)", + "CREATE INDEX commits_attempt ON commits (attempt)", + "CREATE INDEX steps_run ON steps (run, ord)", + ], + &[ + "ALTER TABLE runs ADD COLUMN retry_at INTEGER", + "ALTER TABLE runs ADD COLUMN deferrals INTEGER NOT NULL DEFAULT 0", + ], + &[ + "ALTER TABLE attempts ADD COLUMN peak_rss_bytes INTEGER", + "ALTER TABLE attempts ADD COLUMN cpu_seconds REAL", + "ALTER TABLE attempts ADD COLUMN peak_cpus REAL", + ], + &["ALTER TABLE runs ADD COLUMN token TEXT"], +]; + +pub(crate) struct SqliteStore { + path: String, + db: Connection, +} + +impl SqliteStore { + pub fn open(path: &str) -> Res { + Ok(SqliteStore { + path: path.to_string(), + db: open(path)?, + }) + } +} + +fn now_sql(s: &str) -> String { + s.replace("{NOW}", NOW) +} + +fn still_owner(tx: &rusqlite::Transaction, run: i64, owner: &str) -> Res> { + Ok(tx + .query_row( + &now_sql( + "UPDATE runs SET heartbeat = {NOW} WHERE id = ?1 AND owner = ?2 AND status = \ + 'running' RETURNING cancel_requested", + ), + params![run, owner], + |r| r.get::<_, i64>(0), + ) + .optional()? + .map(|c| c != 0)) +} + +fn fail_running(tx: &rusqlite::Transaction, run: i64, why_a: &str, why_s: &str) -> Res { + let n = tx.execute( + &now_sql( + "UPDATE attempts SET status = 'failed', ended = {NOW}, error = ?2 WHERE status = \ + 'running' AND step IN (SELECT id FROM steps WHERE run = ?1)", + ), + params![run, why_a], + )?; + tx.execute( + "UPDATE steps SET status = 'failed', error = ?2 WHERE run = ?1 AND status = 'running'", + params![run, why_s], + )?; + Ok(n as u64) +} + +impl RunStore for SqliteStore { + fn migrate(&mut self) -> Res<()> { + // BEGIN IMMEDIATE takes the write lock: two processes starting at + // once don't both apply a migration. + let tx = self + .db + .transaction_with_behavior(TransactionBehavior::Immediate)?; + tx.execute( + "CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)", + [], + )?; + let have: i64 = tx.query_row( + "SELECT coalesce(max(version), 0) FROM schema_version", + [], + |r| r.get(0), + )?; + for (i, m) in MIGRATIONS.iter().enumerate() { + let v = i as i64 + 1; + if v <= have { + continue; + } + for s in *m { + tx.execute(s, [])?; + } + tx.execute("INSERT INTO schema_version VALUES (?1)", params![v])?; + } + tx.commit()?; + Ok(()) + } + + fn reconnect(&mut self) -> Res<()> { + self.db = open(&self.path)?; + Ok(()) + } + + fn run_info(&mut self, run: i64) -> Res> { + Ok(self + .db + .query_row( + "SELECT pipeline, registry_hash, partitions, status, coalesce(owner, '') FROM \ + runs WHERE id = ?1", + params![run], + |r| { + Ok(RunInfo { + pipeline: r.get(0)?, + registry_hash: r.get(1)?, + partitions: r.get(2)?, + status: r.get(3)?, + owner: r.get(4)?, + }) + }, + ) + .optional()?) + } + + fn claim(&mut self, run: i64, owner: &str, stale: f64) -> Res { + let got = self + .db + .query_row( + &now_sql( + "UPDATE runs SET status = 'running', owner = ?2, heartbeat = {NOW}, ended = \ + NULL, error = NULL WHERE id = ?1 AND status IN ('queued', 'running', \ + 'failed', 'deferred') AND (heartbeat IS NULL OR heartbeat < {NOW} - ?3) \ + RETURNING id", + ), + params![run, owner, stale], + |r| r.get::<_, i64>(0), + ) + .optional()?; + Ok(got.is_some()) + } + + fn version(&mut self, pipeline: &str, hash: &str, describe: &str, image: &str) -> Res { + Ok(self.db.query_row( + &now_sql( + "INSERT INTO pipeline_versions (pipeline, registry_hash, source, commit_sha, \ + dirty, image, first_seen, last_seen) VALUES (?1, ?2, \ + coalesce(json_extract(?3, '$.source.repo'), ''), coalesce(json_extract(?3, \ + '$.source.commit'), ''), coalesce(json_extract(?3, '$.source.dirty'), 0), ?4, \ + {NOW}, {NOW}) ON CONFLICT (pipeline, registry_hash, source, commit_sha, dirty, \ + image) DO UPDATE SET last_seen = excluded.last_seen RETURNING id", + ), + params![pipeline, hash, describe, image], + |r| r.get(0), + )?) + } + + fn ran_version(&mut self, run: i64, version: i64) -> Res<()> { + self.db.execute( + "UPDATE runs SET version = ?2 WHERE id = ?1", + params![run, version], + )?; + Ok(()) + } + + fn interrupt(&mut self, run: i64, why: &str) -> Res { + let tx = self + .db + .transaction_with_behavior(TransactionBehavior::Immediate)?; + let n = fail_running( + &tx, + run, + if why.is_empty() { INTERRUPTED } else { why }, + if why.is_empty() { "interrupted" } else { why }, + )?; + tx.commit()?; + Ok(n) + } + + fn prior(&mut self, run: i64) -> Res>> { + let mut st = self.db.prepare( + "SELECT step, partition, coalesce(output, '') FROM steps WHERE run = ?1 AND status = \ + 'succeeded' ORDER BY id", + )?; + let rows: Vec<(String, String, String)> = st + .query_map(params![run], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))? + .collect::>()?; + let mut out = HashMap::new(); + for (s, p, o) in rows { + out.insert((s, p), from_hex(&o)?); + } + Ok(out) + } + + fn run_token(&mut self, run: i64) -> Res { + Ok(self + .db + .query_row( + "SELECT coalesce(token, '') FROM runs WHERE id = ?1", + params![run], + |r| r.get(0), + ) + .optional()? + .unwrap_or_default()) + } + + fn start_attempt( + &mut self, + run: i64, + owner: &str, + hash: &str, + step: &str, + partition: &str, + ord: i32, + input: &[u8], + ) -> Res { + let input = to_hex(input); + let tx = self + .db + .transaction_with_behavior(TransactionBehavior::Immediate)?; + match still_owner(&tx, run, owner)? { + None => return Err(lost(run)), + Some(true) => return Err(cancelled(run)), + Some(false) => {} + } + let step_id: i64 = tx.query_row( + "INSERT INTO steps (run, step, partition, ord, status, input) VALUES (?1, ?2, ?3, \ + ?4, 'running', ?5) ON CONFLICT (run, step, partition) DO UPDATE SET status = \ + 'running', input = excluded.input, error = NULL RETURNING id", + params![run, step, partition, ord, input], + |r| r.get(0), + )?; + let number: i64 = tx.query_row( + "SELECT coalesce(max(attempt), 0) + 1 FROM attempts WHERE step = ?1", + params![step_id], + |r| r.get(0), + )?; + let id: i64 = tx.query_row( + &now_sql( + "INSERT INTO attempts (step, attempt, status, host, registry_hash, started) \ + VALUES (?1, ?2, 'running', ?3, ?4, {NOW}) RETURNING id", + ), + params![step_id, number, owner, hash], + |r| r.get(0), + )?; + tx.commit()?; + Ok(Attempt { number, id }) + } + + fn finish_attempt( + &mut self, + run: i64, + owner: &str, + step: &str, + partition: &str, + attempt: i64, + status: &str, + output: &[u8], + error: &str, + ) -> Res<()> { + let output = to_hex(output); + let tx = self + .db + .transaction_with_behavior(TransactionBehavior::Immediate)?; + if still_owner(&tx, run, owner)?.is_none() { + return Err(lost(run)); + } + let step_id: i64 = tx.query_row( + "SELECT id FROM steps WHERE run = ?1 AND step = ?2 AND partition = ?3", + params![run, step, partition], + |r| r.get(0), + )?; + tx.execute( + &now_sql( + "UPDATE attempts SET status = ?3, ended = {NOW}, error = nullif(?4, '') WHERE \ + step = ?1 AND attempt = ?2", + ), + params![step_id, attempt, status, error], + )?; + let out: Option<&str> = if status == super::SUCCEEDED { + Some(&output) + } else { + None + }; + tx.execute( + "UPDATE steps SET status = ?2, output = ?3, error = nullif(?4, '') WHERE id = ?1", + params![step_id, status, out, error], + )?; + tx.commit()?; + Ok(()) + } + + fn skip( + &mut self, + run: i64, + owner: &str, + step: &str, + partition: &str, + ord: i32, + reason: &str, + ) -> Res<()> { + let tx = self + .db + .transaction_with_behavior(TransactionBehavior::Immediate)?; + match still_owner(&tx, run, owner)? { + None => return Err(lost(run)), + Some(true) => return Err(cancelled(run)), + Some(false) => {} + } + tx.execute( + "INSERT INTO steps (run, step, partition, ord, status, error) VALUES (?1, ?2, ?3, \ + ?4, 'skipped', ?5) ON CONFLICT (run, step, partition) DO UPDATE SET status = \ + 'skipped', error = excluded.error, output = NULL", + params![run, step, partition, ord, reason], + )?; + tx.commit()?; + Ok(()) + } + + fn set_log(&mut self, attempt_id: i64, location: &str) -> Res<()> { + self.db.execute( + "UPDATE attempts SET log = ?2 WHERE id = ?1", + params![attempt_id, location], + )?; + Ok(()) + } + + fn pinned_reads(&mut self, run: i64, step: &str, partition: &str) -> Res> { + let mut st = self.db.prepare( + "SELECT r.table_name, r.snapshot_id FROM reads r JOIN attempts a ON a.id = r.attempt \ + JOIN steps s ON s.id = a.step WHERE s.run = ?1 AND s.step = ?2 AND s.partition = ?3 \ + ORDER BY a.attempt DESC", + )?; + let rows: Vec<(String, i64)> = st + .query_map(params![run, step, partition], |r| { + Ok((r.get(0)?, r.get(1)?)) + })? + .collect::>()?; + Ok(rows.into_iter().collect()) + } + + fn record( + &mut self, + commit: bool, + attempt_id: i64, + table: &str, + snapshot: i64, + dataset: &str, + ) -> Res<()> { + let sql = format!( + "INSERT INTO {} (attempt, table_name, snapshot_id, dataset) VALUES (?1, ?2, ?3, \ + nullif(?4, ''))", + if commit { "commits" } else { "reads" } + ); + self.db + .execute(&sql, params![attempt_id, table, snapshot, dataset])?; + Ok(()) + } + + fn fence(&mut self, run: i64, fresh_seconds: f64) -> Res> { + Ok(self + .db + .query_row( + &now_sql( + "SELECT coalesce(owner, ''), status, coalesce(heartbeat > {NOW} - ?2, 0) \ + FROM runs WHERE id = ?1", + ), + params![run, fresh_seconds], + |r| { + Ok(Fence { + owner: r.get(0)?, + status: r.get(1)?, + fresh: r.get::<_, i64>(2)? != 0, + }) + }, + ) + .optional()?) + } + + fn cancel_requested(&mut self, run: i64) -> Res { + Ok(self + .db + .query_row( + "SELECT cancel_requested FROM runs WHERE id = ?1", + params![run], + |r| r.get::<_, i64>(0), + ) + .optional()? + .is_some_and(|c| c != 0)) + } + + fn finish_run(&mut self, run: i64, owner: &str, status: &str) -> Res<()> { + self.db.execute( + &now_sql( + "UPDATE runs SET status = ?2, ended = {NOW}, heartbeat = NULL WHERE id = ?1 AND \ + owner = ?3", + ), + params![run, status, owner], + )?; + Ok(()) + } + + fn defer_run(&mut self, run: i64, owner: &str, until: i64) -> Res<()> { + self.db.execute( + &now_sql( + "UPDATE runs SET status = 'deferred', ended = {NOW}, heartbeat = NULL, retry_at = \ + ?3, deferrals = deferrals + 1 WHERE id = ?1 AND owner = ?2", + ), + params![run, owner, until], + )?; + Ok(()) + } + + fn beater(&self, run: i64, owner: &str, timeouts: &str) -> Box { + Box::new(SqliteBeat { + path: self.path.clone(), + run, + owner: owner.to_string(), + timeouts: timeouts.to_string(), + db: None, + }) + } +} + +/// The heartbeat over the file. There's no cancel to time out here: a +/// cancel is the scheduler's, on Postgres. +struct SqliteBeat { + path: String, + run: i64, + owner: String, + timeouts: String, + db: Option, +} + +impl SqliteBeat { + fn timed_out(&mut self) -> Res> { + if self.timeouts.is_empty() { + return Ok(None); + } + let (run, owner) = (self.run, self.owner.clone()); + let pairs: Vec<(String, f64)> = self + .timeouts + .split(',') + .filter_map(|p| { + let (s, t) = p.split_once('=')?; + Some((s.to_string(), t.parse().ok()?)) + }) + .collect(); + let db = self.db.as_mut().unwrap(); + for (step, secs) in pairs { + let got: Option<(i64, String)> = db + .query_row( + &now_sql( + "SELECT a.id, s.partition FROM attempts a JOIN steps s ON s.id = a.step \ + JOIN runs r ON r.id = s.run WHERE s.run = ?1 AND s.step = ?2 AND \ + r.owner = ?3 AND r.status = 'running' AND a.status = 'running' AND \ + a.started <= {NOW} - ?4 LIMIT 1", + ), + params![run, step, owner, secs], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional()?; + let Some((attempt, partition)) = got else { + continue; + }; + let why = heartbeat::timed_out_why(secs); + let tx = db.transaction_with_behavior(TransactionBehavior::Immediate)?; + let n = tx.execute( + &now_sql( + "UPDATE attempts SET status = 'failed', ended = {NOW}, error = ?2 WHERE id = \ + ?1 AND status = 'running'", + ), + params![attempt, why], + )?; + if n != 1 { + return Ok(None); // it ended as we looked + } + tx.execute( + "UPDATE steps SET status = 'failed', error = ?2 WHERE id = (SELECT step FROM \ + attempts WHERE id = ?1)", + params![attempt, why], + )?; + let named = if partition.is_empty() { + step + } else { + format!("{step} {partition}") + }; + let run_why = format!("step {named} {why}"); + tx.execute( + &now_sql( + "UPDATE runs SET status = 'failed', ended = {NOW}, heartbeat = NULL, error = \ + ?3 WHERE id = ?1 AND owner = ?2", + ), + params![run, owner, run_why], + )?; + tx.commit()?; + return Ok(Some(run_why)); + } + Ok(None) + } +} + +impl Beater for SqliteBeat { + fn beat(&mut self) -> Result, String> { + if self.db.is_none() { + self.db = Some(open(&self.path).map_err(|e| e.to_string())?); + } + let (run, owner) = (self.run, self.owner.clone()); + self.db + .as_ref() + .unwrap() + .execute( + &now_sql( + "UPDATE runs SET heartbeat = {NOW} WHERE id = ?1 AND owner = ?2 AND status = \ + 'running'", + ), + params![run, owner], + ) + .map_err(|e| e.to_string())?; + self.timed_out().map_err(|e| e.to_string()) + } + + fn reset(&mut self) { + self.db = None; + } +} diff --git a/rust/morca-sdk/src/tables/catalog.rs b/rust/morca-sdk/src/tables/catalog.rs new file mode 100644 index 00000000..f871a69d --- /dev/null +++ b/rust/morca-sdk/src/tables/catalog.rs @@ -0,0 +1,487 @@ +//! morca's filesystem catalog, in Rust (from the iceberg-rust spike, #520, +//! spikes/iceberg-rust/src/catalog.rs): iceberg-rust's `Catalog` over +//! `object_store`, committing the way iceberg-mojo's `FilesystemCatalog` +//! does (iceberg.mojo `src/iceberg/catalog/filesystem.mojo`, 0.12.0). +//! +//! A dataset `ns.name` is the table at `/ns/name`. Its current +//! metadata is the newest `-.metadata.json` in `metadata/`: +//! `version-hint.text` names a version to start from, and the newest is +//! found by stepping forward a version at a time with one prefix listing +//! each (`-`); with no hint, by listing the directory and taking the +//! highest version, ties broken by `last-updated-ms`. +//! +//! A commit of version `V`: +//! +//! 1. refuses if a file of `V` exists, or the hint is at `V` or past it +//! (a stale base); +//! 2. writes a claim `-..claim` (empty); +//! 3. lists `-`: any other claim younger than a minute, or any +//! metadata file of `V`, is a rival; claims older than that are removed; +//! 4. with a rival, removes its claim and fails (retryable: the transaction +//! reloads and tries again); with none, writes `-.metadata.json`, +//! removes its claim, and writes `version-hint.text`. +//! +//! Every write here is `PutMode::Create`, which on S3 is `If-None-Match: *`; +//! iceberg-mojo checks existence and then puts. The names are unique, so +//! neither relies on it; the listing is what settles a race, and needs a +//! store whose listings see every write before them (local disk, S3 since +//! 2020, MinIO). +//! +//! Data and manifest files go through iceberg-rust's own `FileIO`. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use bytes::Bytes; +use iceberg::io::FileIO; +use iceberg::spec::{TableMetadata, TableMetadataBuilder}; +use iceberg::table::Table; +use iceberg::{ + Catalog, Error, ErrorKind, Namespace, NamespaceIdent, Result, Runtime, TableCommit, + TableCreation, TableIdent, +}; +use object_store::aws::AmazonS3; +use object_store::list::{PaginatedListOptions, PaginatedListStore}; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutOptions, PutPayload}; + +const METADATA_SUFFIX: &str = ".metadata.json"; +const CLAIM_SUFFIX: &str = ".claim"; +const VERSION_HINT: &str = "version-hint.text"; +/// As iceberg-mojo's `CLAIM_STALE_MS`. +const CLAIM_STALE_MS: i64 = 60_000; +/// As iceberg-mojo's `CREATE_WAIT_MS`. +const CREATE_WAIT_MS: i64 = 10_000; + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() as i64 +} + +fn os_err(e: object_store::Error) -> Error { + Error::new(ErrorKind::Unexpected, "object store").with_source(e) +} + +fn conflict(msg: String) -> Error { + Error::new(ErrorKind::CatalogCommitConflicts, msg).with_retryable(true) +} + +/// The leading version of a metadata file name, or None. +fn version_of(name: &str) -> Option { + if !name.ends_with(METADATA_SUFFIX) { + return None; + } + let s = name.strip_prefix('v').unwrap_or(name); + let digits: String = s.chars().take_while(|c| c.is_ascii_digit()).collect(); + digits.parse().ok() +} + +/// When a claim was made, from its name, or None if it isn't one. +fn claim_ms(name: &str) -> Option { + let stem = name.strip_suffix(CLAIM_SUFFIX)?; + stem.rsplit_once('.')?.1.parse().ok() +} + +#[derive(Debug)] +pub(crate) struct MorcaCatalog { + /// The warehouse root as tables' `location`s spell it: a bare absolute + /// path, or `s3://bucket/prefix`. + warehouse: String, + store: Arc, + /// The same store, for listings by key prefix, on S3. + s3: Option>, + /// `s3://bucket/` (with the slash) for S3, "" for local disk. + scheme_root: String, + file_io: FileIO, +} + +impl MorcaCatalog { + pub fn new( + warehouse: String, + store: Arc, + s3: Option>, + file_io: FileIO, + ) -> Self { + let warehouse = warehouse.trim_end_matches('/').to_string(); + let scheme_root = match warehouse.strip_prefix("s3://") { + Some(rest) => format!("s3://{}/", rest.split('/').next().unwrap()), + None => String::new(), + }; + Self { + warehouse, + store, + s3, + scheme_root, + file_io, + } + } + + /// `ns.name` -> `/ns/name`. + pub fn table_location(&self, ident: &TableIdent) -> String { + format!( + "{}/{}/{}", + self.warehouse, + ident.namespace().join("/"), + ident.name() + ) + } + + fn path(&self, location: &str) -> Path { + let key = if self.scheme_root.is_empty() { + location + .trim_start_matches("file://") + .trim_start_matches('/') + } else { + location.strip_prefix(&self.scheme_root).unwrap_or(location) + }; + Path::from(key) + } + + /// Names in `dir` starting with `prefix` (one listing). + async fn names_with_prefix(&self, dir: &str, prefix: &str) -> Result> { + if self.scheme_root.is_empty() { + // One readdir, as iceberg-mojo's local listing (`std.os.listdir`): + // object_store's LocalFileSystem walks and stats every entry, + // which widens the window in which a listing misses a file + // created, and a claim removed, while it runs (spikes/iceberg-rust/README.md). + let names = match std::fs::read_dir(dir) { + Ok(rd) => rd + .filter_map(|e| e.ok()) + .filter_map(|e| e.file_name().into_string().ok()) + .filter(|n| n.starts_with(prefix)) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => vec![], + Err(e) => return Err(Error::new(ErrorKind::Unexpected, "listing").with_source(e)), + }; + return Ok(names); + } + // On S3, ListObjectsV2 with the key prefix `/`, as + // iceberg-mojo lists: one short page. Listing the directory from an + // offset (start-after) lost commits on MinIO (spikes/iceberg-rust/README.md). + let s3 = self.s3.as_ref().ok_or_else(|| unsupported("listing"))?; + let key_prefix = format!("{}/{prefix}", self.path(dir).as_ref()); + let mut names = vec![]; + let mut token = None; + loop { + let page = s3 + .list_paginated( + Some(&key_prefix), + PaginatedListOptions { + page_token: token, + ..Default::default() + }, + ) + .await + .map_err(os_err)?; + names.extend( + page.result + .objects + .into_iter() + .filter_map(|m| m.location.filename().map(str::to_string)), + ); + token = page.page_token; + if token.is_none() { + return Ok(names); + } + } + } + + async fn read_hint(&self, dir: &str) -> Option { + let got = self + .store + .get(&self.path(&format!("{dir}/{VERSION_HINT}"))) + .await + .ok()?; + let b = got.bytes().await.ok()?; + String::from_utf8_lossy(&b).trim().parse().ok() + } + + async fn read_metadata(&self, location: &str) -> Result { + let got = self.store.get(&self.path(location)).await.map_err(os_err)?; + let b = got.bytes().await.map_err(os_err)?; + Ok(serde_json::from_slice(&b)?) + } + + /// One version's metadata files, as locations. + async fn files_of_version(&self, dir: &str, v: i64) -> Result> { + Ok(self + .names_with_prefix(dir, &format!("{v:05}-")) + .await? + .into_iter() + .filter(|n| version_of(n) == Some(v)) + .map(|n| format!("{dir}/{n}")) + .collect()) + } + + /// Of several files of one version, the one with the latest + /// `last-updated-ms`. + async fn newest_of(&self, mut paths: Vec) -> Option { + paths.sort(); + if paths.len() <= 1 { + return paths.pop(); + } + let mut best = (i64::MIN, paths[0].clone()); + for p in paths { + if let Ok(m) = self.read_metadata(&p).await + && m.last_updated_ms() > best.0 + { + best = (m.last_updated_ms(), p); + } + } + Some(best.1) + } + + /// The current metadata file of the table at `location`, or None. + pub async fn find_latest_metadata(&self, location: &str) -> Result> { + let dir = format!("{location}/metadata"); + if let Some(hint) = self.read_hint(&dir).await + && let Some(mut found) = self + .newest_of(self.files_of_version(&dir, hint).await?) + .await + { + let mut v = hint + 1; + while let Some(next) = self.newest_of(self.files_of_version(&dir, v).await?).await { + found = next; + v += 1; + } + return Ok(Some(found)); + } + let names = self.names_with_prefix(&dir, "").await?; + let Some(best) = names.iter().filter_map(|n| version_of(n)).max() else { + return Ok(None); + }; + let candidates = names + .into_iter() + .filter(|n| version_of(n) == Some(best)) + .map(|n| format!("{dir}/{n}")) + .collect(); + Ok(self.newest_of(candidates).await) + } + + async fn put_new(&self, location: &str, data: Bytes) -> Result<()> { + self.store + .put_opts( + &self.path(location), + PutPayload::from(data), + PutOptions { + mode: PutMode::Create, + ..Default::default() + }, + ) + .await + .map_err(os_err)?; + Ok(()) + } + + async fn remove(&self, location: &str) { + let _ = self.store.delete(&self.path(location)).await; + } + + /// Other claims on `v` and its metadata files (stale claims removed). + async fn rivals(&self, dir: &str, v: i64, mine: &str) -> Result { + let now = now_ms(); + let mut n = 0; + for name in self.names_with_prefix(dir, &format!("{v:05}-")).await? { + if name == mine { + continue; + } + if version_of(&name) == Some(v) { + n += 1; + } else if let Some(at) = claim_ms(&name) { + if now - at > CLAIM_STALE_MS { + self.remove(&format!("{dir}/{name}")).await; + } else { + n += 1; + } + } + } + Ok(n) + } + + fn claim_name(v: i64) -> String { + format!("{v:05}-{}.{}{CLAIM_SUFFIX}", uuid::Uuid::new_v4(), now_ms()) + } + + /// Writes metadata version `v` (the file `path`) under a claim, as + /// iceberg-mojo's `Table.commit_metadata` does. `Ok(false)`: a rival + /// claimed it too, and nothing was written. + async fn claim_and_write(&self, dir: &str, v: i64, path: &str, doc: Bytes) -> Result { + let claim = Self::claim_name(v); + let claim_path = format!("{dir}/{claim}"); + self.put_new(&claim_path, Bytes::new()).await?; + let rivals = match self.rivals(dir, v, &claim).await { + Ok(n) => n, + Err(e) => { + self.remove(&claim_path).await; + return Err(e); + } + }; + if rivals > 0 { + self.remove(&claim_path).await; + return Ok(false); + } + let wrote = self.put_new(path, doc).await; + self.remove(&claim_path).await; + wrote?; + // Advisory, after the metadata file, as iceberg-mojo writes it. + let _ = self + .store + .put( + &self.path(&format!("{dir}/{VERSION_HINT}")), + PutPayload::from(format!("{v}\n")), + ) + .await; + Ok(true) + } + + fn table(&self, ident: TableIdent, location: String, metadata: TableMetadata) -> Result { + Table::builder() + .file_io(self.file_io.clone()) + .metadata_location(location) + .metadata(metadata) + .identifier(ident) + .runtime(Runtime::current()) + .build() + } +} + +fn unsupported(what: &str) -> Error { + Error::new( + ErrorKind::FeatureUnsupported, + format!("MorcaCatalog: {what}"), + ) +} + +#[async_trait] +impl Catalog for MorcaCatalog { + async fn list_namespaces(&self, _: Option<&NamespaceIdent>) -> Result> { + Err(unsupported("list_namespaces")) + } + async fn create_namespace( + &self, + ns: &NamespaceIdent, + _: HashMap, + ) -> Result { + // A namespace is a directory: nothing to create. + Ok(Namespace::new(ns.clone())) + } + async fn get_namespace(&self, ns: &NamespaceIdent) -> Result { + Ok(Namespace::new(ns.clone())) + } + async fn namespace_exists(&self, _: &NamespaceIdent) -> Result { + Ok(true) + } + async fn update_namespace(&self, _: &NamespaceIdent, _: HashMap) -> Result<()> { + Err(unsupported("update_namespace")) + } + async fn drop_namespace(&self, _: &NamespaceIdent) -> Result<()> { + Err(unsupported("drop_namespace")) + } + async fn list_tables(&self, _: &NamespaceIdent) -> Result> { + Err(unsupported("list_tables")) + } + + /// Writes version 0 under a claim, as iceberg-mojo's + /// `write_first_metadata`. A creator that loses waits for the + /// winner's table and returns it. + async fn create_table(&self, ns: &NamespaceIdent, creation: TableCreation) -> Result
{ + let ident = TableIdent::new(ns.clone(), creation.name.clone()); + let location = self.table_location(&ident); + let creation = TableCreation { + location: Some(location.clone()), + ..creation + }; + let metadata = TableMetadataBuilder::from_table_creation(creation)? + .build()? + .metadata; + let dir = format!("{location}/metadata"); + let path = format!("{dir}/00000-{}{METADATA_SUFFIX}", uuid::Uuid::new_v4()); + let doc = Bytes::from(serde_json::to_vec(&metadata)?); + let deadline = now_ms() + CREATE_WAIT_MS; + loop { + if self.claim_and_write(&dir, 0, &path, doc.clone()).await? { + return self.table(ident, path, metadata); + } + loop { + if !self.files_of_version(&dir, 0).await?.is_empty() { + return self.load_table(&ident).await; + } + if self.rivals(&dir, 0, "").await? == 0 { + break; + } + if now_ms() > deadline { + return Err(conflict(format!( + "another writer is still creating {location}; retry" + ))); + } + tokio::time::sleep(Duration::from_millis(1 + (now_ms() % 10) as u64)).await; + } + } + } + + async fn load_table(&self, ident: &TableIdent) -> Result
{ + let location = self.table_location(ident); + let Some(path) = self.find_latest_metadata(&location).await? else { + return Err(Error::new( + ErrorKind::TableNotFound, + format!("no table at {location}"), + )); + }; + let metadata = self.read_metadata(&path).await?; + self.table(ident.clone(), path, metadata) + } + + async fn drop_table(&self, _: &TableIdent) -> Result<()> { + Err(unsupported("drop_table")) + } + async fn purge_table(&self, _: &TableIdent) -> Result<()> { + Err(unsupported("purge_table")) + } + async fn table_exists(&self, ident: &TableIdent) -> Result { + Ok(self + .find_latest_metadata(&self.table_location(ident)) + .await? + .is_some()) + } + async fn rename_table(&self, _: &TableIdent, _: &TableIdent) -> Result<()> { + Err(unsupported("rename_table")) + } + async fn register_table(&self, _: &TableIdent, _: String) -> Result
{ + Err(unsupported("register_table")) + } + + async fn update_table(&self, commit: TableCommit) -> Result
{ + let current = self.load_table(commit.identifier()).await?; + // Checks the requirements (the table's UUID, main's snapshot, ...) + // against what's current, applies the updates, and names the next + // version `-.metadata.json`. + let staged = commit.apply(current).map_err(|e| { + if e.kind() == ErrorKind::CatalogCommitConflicts { + e.with_retryable(true) + } else { + e + } + })?; + let path = staged.metadata_location_result()?.to_string(); + let (dir, name) = path.rsplit_once('/').unwrap(); + let v = version_of(name).unwrap(); + if !self.files_of_version(dir, v).await?.is_empty() + || self.read_hint(dir).await.is_some_and(|h| h >= v) + { + return Err(conflict(format!( + "another writer committed version {v} first" + ))); + } + let doc = Bytes::from(serde_json::to_vec(staged.metadata())?); + if !self.claim_and_write(dir, v, &path, doc).await? { + return Err(conflict(format!( + "several writers claimed version {v}; reload and retry" + ))); + } + Ok(staged) + } +} diff --git a/rust/morca-sdk/src/tables/imp.rs b/rust/morca-sdk/src/tables/imp.rs new file mode 100644 index 00000000..0d5db1d0 --- /dev/null +++ b/rust/morca-sdk/src/tables/imp.rs @@ -0,0 +1,531 @@ +use std::collections::HashMap; +use std::future::Future; +use std::sync::{Arc, OnceLock}; + +pub use arrow_array; +pub use arrow_schema; +pub use iceberg; + +use arrow_array::{Array, RecordBatch, StringArray}; +use arrow_schema::SchemaRef; +use futures::TryStreamExt; +use iceberg::arrow::{RecordBatchPartitionSplitter, schema_to_arrow_schema}; +use iceberg::io::{ + FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, + S3_SECRET_ACCESS_KEY, +}; +use iceberg::spec::{DataFile, DataFileFormat, Schema, TableMetadata, UnboundPartitionSpec}; +use iceberg::table::Table; +use iceberg::transaction::{ApplyTransactionAction, Transaction}; +use iceberg::writer::base_writer::data_file_writer::DataFileWriterBuilder; +use iceberg::writer::file_writer::ParquetWriterBuilder; +use iceberg::writer::file_writer::location_generator::{ + DefaultFileNameGenerator, DefaultLocationGenerator, +}; +use iceberg::writer::file_writer::rolling_writer::RollingFileWriterBuilder; +use iceberg::writer::{IcebergWriter, IcebergWriterBuilder}; +use iceberg::{Catalog, ErrorKind, NamespaceIdent, TableCreation, TableIdent}; +use iceberg_storage_opendal::OpenDalStorageFactory; +use object_store::ObjectStore; +use object_store::aws::AmazonS3Builder; +use object_store::local::LocalFileSystem; +use parquet::file::properties::WriterProperties; + +use super::catalog::MorcaCatalog; +use crate::ctx::Ctx; +use crate::pipeline::{Built, StepResult, fail}; + +fn runtime() -> &'static tokio::runtime::Runtime { + static RT: OnceLock = OnceLock::new(); + RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .expect("morca: can't start tokio") + }) +} + +fn block(f: F) -> F::Output { + runtime().block_on(f) +} + +fn env(k: &str) -> Option { + std::env::var(k).ok().filter(|v| !v.is_empty()) +} + +/// The catalog over `MORCA_WAREHOUSE`: local disk, or S3 from `AWS_*` +/// (`AWS_ENDPOINT_URL` for MinIO). +pub(crate) fn catalog() -> Result { + let wh = env("MORCA_WAREHOUSE").ok_or("morca: MORCA_WAREHOUSE is unset")?; + let wh = wh.strip_prefix("file://").map(str::to_string).unwrap_or(wh); + if let Some(rest) = wh.strip_prefix("s3://") { + let bucket = rest.split('/').next().unwrap_or("").to_string(); + let region = env("AWS_REGION") + .or_else(|| env("AWS_DEFAULT_REGION")) + .unwrap_or_else(|| "us-east-1".into()); + let key = env("AWS_ACCESS_KEY_ID").ok_or("morca: AWS_ACCESS_KEY_ID is unset")?; + let secret = env("AWS_SECRET_ACCESS_KEY").ok_or("morca: AWS_SECRET_ACCESS_KEY is unset")?; + let mut s3 = AmazonS3Builder::new() + .with_bucket_name(&bucket) + .with_region(®ion) + .with_access_key_id(&key) + .with_secret_access_key(&secret); + let mut props = vec![ + (S3_REGION, region.clone()), + (S3_ACCESS_KEY_ID, key), + (S3_SECRET_ACCESS_KEY, secret), + ]; + if let Some(t) = env("AWS_SESSION_TOKEN") { + s3 = s3.with_token(&t); + } + if let Some(ep) = env("AWS_ENDPOINT_URL") { + s3 = s3 + .with_endpoint(&ep) + .with_allow_http(ep.starts_with("http://")) + .with_virtual_hosted_style_request(false); + props.push((S3_ENDPOINT, ep)); + props.push((S3_PATH_STYLE_ACCESS, "true".into())); + } + let s3 = Arc::new(s3.build().map_err(|e| format!("morca: S3: {e}"))?); + let store: Arc = s3.clone(); + let io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_props(props) + .build(); + return Ok(MorcaCatalog::new(wh, store, Some(s3), io)); + } + let abs = if wh.starts_with('/') { + wh + } else { + let cwd = std::env::current_dir().map_err(|e| e.to_string())?; + format!("{}/{wh}", cwd.display()) + }; + Ok(MorcaCatalog::new( + abs, + Arc::new(LocalFileSystem::new()), + None, + FileIO::new_with_fs(), + )) +} + +pub(crate) fn ident(dataset: &str) -> Result { + let (ns, name) = dataset + .rsplit_once('.') + .ok_or_else(|| format!("morca: '{dataset}' isn't namespace.table"))?; + Ok(TableIdent::new( + NamespaceIdent::from_strs(ns.split('.')).map_err(|e| e.to_string())?, + name.to_string(), + )) +} + +/// The table, or None if there's none. +async fn load(cat: &MorcaCatalog, id: &TableIdent) -> Result, String> { + match cat.load_table(id).await { + Ok(t) => Ok(Some(t)), + Err(e) if e.kind() == ErrorKind::TableNotFound => Ok(None), + Err(e) => Err(format!("morca: can't load {id}: {e}")), + } +} + +fn current(md: &TableMetadata) -> i64 { + md.current_snapshot_id().unwrap_or(-1) +} + +/// `table_defaults()` in src/morca/_lake/names.mojo. +fn table_defaults() -> HashMap { + HashMap::from([ + ( + "write.metadata.delete-after-commit.enabled".into(), + "true".into(), + ), + ("write.metadata.previous-versions-max".into(), "100".into()), + ( + "history.expire.max-snapshot-age-ms".into(), + (30i64 * 86_400_000).to_string(), + ), + ]) +} + +/// How many times to reload and retry when another writer got there first +/// (`_COMMIT_RETRIES` in src/morca/_lake/impl.mojo). +const COMMIT_RETRIES: u32 = 8; + +impl Ctx<'_> { + fn declared(&self, dataset: &str, write: bool) -> StepResult<()> { + let list = if write { + &self.declared_writes + } else { + &self.declared_reads + }; + if !list.iter().any(|d| d == dataset) { + return Err(fail(format!( + "morca: step {} doesn't declare that it {} {dataset}: add it to its {}", + self.step, + if write { "writes" } else { "reads" }, + if write { "writes" } else { "reads" }, + ))); + } + Ok(()) + } + + /// The Arrow schema of the dataset's current schema, with Iceberg's + /// field ids: what the batches handed to [`Ctx::append`] must have. + pub fn arrow_schema(&mut self, dataset: &str) -> StepResult { + let cat = catalog()?; + let t = block(load(&cat, &ident(dataset)?))? + .ok_or_else(|| fail(format!("morca: there's no table {dataset}")))?; + Ok(Arc::new(schema_to_arrow_schema( + t.metadata().current_schema(), + )?)) + } + + /// Creates the dataset's table with `schema` (and `spec`, else + /// unpartitioned) if there's none, with morca's table defaults. A + /// table that exists is left as it is. + pub fn ensure_table( + &mut self, + dataset: &str, + schema: Schema, + spec: Option, + ) -> StepResult<()> { + self.declared(dataset, true)?; + let cat = catalog()?; + let id = ident(dataset)?; + block(async { + if load(&cat, &id).await?.is_some() { + return Ok::<(), String>(()); + } + let mut b = TableCreation::builder() + .name(id.name().to_string()) + .schema(schema) + .properties(table_defaults()) + .build(); + if let Some(s) = spec { + b.partition_spec = Some(s); + } + cat.create_table(id.namespace(), b) + .await + .map_err(|e| format!("morca: can't create {dataset}: {e}"))?; + Ok(()) + })?; + Ok(()) + } + + /// The rows of the dataset at the snapshot this step reads (pinned on a + /// retry or a resume), recorded as this attempt's read. No table, or + /// one with no snapshot, is no rows. + pub fn read(&mut self, dataset: &str) -> StepResult> { + self.declared(dataset, false)?; + let cat = catalog()?; + let id = ident(dataset)?; + let t = block(load(&cat, &id))?; + let (location, cur) = match &t { + Some(t) => (t.metadata().location().to_string(), current(t.metadata())), + None => (cat.table_location(&id), -1), + }; + let snap = self.pin(&location, cur); + let batches = match (&t, snap) { + (Some(t), s) if s >= 0 => block(async { + let scan = t + .scan() + .select_all() + .snapshot_id(s) + .build() + .map_err(|e| e.to_string())?; + scan.to_arrow() + .await + .map_err(|e| e.to_string())? + .try_collect::>() + .await + .map_err(|e| e.to_string()) + }) + .map_err(|e| fail(format!("morca: can't read {dataset} at {s}: {e}")))?, + _ => vec![], + }; + self.record_read(&location, snap, dataset)?; + Ok(batches) + } + + /// The snapshot an earlier attempt of this run made as this attempt's + /// next commit to the dataset, or None (`find_commit`). A step that + /// writes files before [`Ctx::append`] calls it first, so its retry + /// doesn't write them again. + pub fn find_commit(&mut self, dataset: &str) -> StepResult> { + let cat = catalog()?; + let Some(t) = block(load(&cat, &ident(dataset)?))? else { + return Ok(None); + }; + Ok(self.found(t.metadata())) + } + + fn found(&self, md: &TableMetadata) -> Option { + let loc = md.location(); + let mut snaps: Vec<_> = md.snapshots().collect(); + snaps.sort_by_key(|s| s.sequence_number()); + snaps + .into_iter() + .find(|s| self.made(loc, &s.summary().additional_properties)) + .map(|s| s.snapshot_id()) + } + + /// Appends `batches` to the dataset in one snapshot, and returns it: + /// written as Parquet split by the table's partition spec, fenced, + /// committed with the `orca.*` summary and recorded. If an earlier + /// attempt of this run already made this commit, records that one and + /// writes nothing. + pub fn append(&mut self, dataset: &str, batches: Vec) -> StepResult { + self.declared(dataset, true)?; + let cat = catalog()?; + let id = ident(dataset)?; + let mut t = block(load(&cat, &id))?.ok_or_else(|| { + fail(format!( + "morca: there's no table {dataset}: ensure_table it first" + )) + })?; + let location = t.metadata().location().to_string(); + if let Some(found) = self.found(t.metadata()) { + self.log_to_file(&format!( + "found commit {found} to {dataset} from an earlier attempt" + )); + self.record_commit(&location, found, dataset)?; + return Ok(found); + } + let files = block(write_files(&t, &batches)) + .map_err(|e| fail(format!("morca: can't write {dataset}'s files: {e}")))?; + let mut tries = 0; + let snap = loop { + self.fence()?; + let summary = self.summary(&location); + let tx = Transaction::new(&t); + let tx = tx + .fast_append() + .add_data_files(files.clone()) + .set_snapshot_properties(summary) + .apply(tx)?; + match block(tx.commit(&cat)) { + Ok(t2) => break current(t2.metadata()), + Err(e) if e.retryable() && tries < COMMIT_RETRIES => { + tries += 1; + let base = 0.01 * f64::from(1u32 << (tries - 1).min(6)); + let wait = base * rand::random_range(0.5..1.5); + std::thread::sleep(std::time::Duration::from_secs_f64(wait)); + t = block(load(&cat, &id))? + .ok_or_else(|| fail(format!("morca: {dataset} went away")))?; + if let Some(found) = self.found(t.metadata()) { + break found; + } + } + Err(e) => { + return Err(fail(format!( + "morca: can't commit to {dataset} ({} tries): {e}", + tries + 1 + ))); + } + } + }; + self.record_commit(&location, snap, dataset)?; + Ok(snap) + } +} + +/// Writes `batches` into the table's `data/`, a file per partition. +async fn write_files(t: &Table, batches: &[RecordBatch]) -> iceberg::Result> { + let md = t.metadata(); + let schema = md.current_schema().clone(); + let spec = md.default_partition_spec().clone(); + let builder = DataFileWriterBuilder::new(RollingFileWriterBuilder::new_with_default_file_size( + ParquetWriterBuilder::new(WriterProperties::builder().build(), schema.clone()), + t.file_io().clone(), + DefaultLocationGenerator::new(md)?, + DefaultFileNameGenerator::new( + uuid::Uuid::new_v4().to_string(), + None, + DataFileFormat::Parquet, + ), + )); + let mut files = vec![]; + if spec.is_unpartitioned() { + let mut w = builder.build(None).await?; + for b in batches { + w.write(b.clone()).await?; + } + files.extend(w.close().await?); + return Ok(files); + } + let splitter = RecordBatchPartitionSplitter::try_new_with_computed_values(schema, spec)?; + for b in batches { + for (key, part) in splitter.split(b)? { + let mut w = builder.build(Some(key)).await?; + w.write(part).await?; + files.extend(w.close().await?); + } + } + Ok(files) +} + +/// `sense`: one line per sensed dataset, ` +/// `, from its current snapshot. +pub(crate) fn sense(p: &Built) -> i32 { + let sensed = p.sensed(); + if sensed.is_empty() { + return 0; + } + let cat = match catalog() { + Ok(c) => c, + Err(e) => { + eprintln!("{e}"); + return 3; + } + }; + for ds in sensed { + let got = ident(&ds).and_then(|id| block(load(&cat, &id))); + let t = match got { + Ok(t) => t, + Err(e) => { + eprintln!("{e}"); + return 3; + } + }; + let (snap, by, parts) = match t.as_ref().and_then(|t| t.metadata().current_snapshot()) { + None => (-1, "other".to_string(), "-".to_string()), + Some(s) => { + let sum = &s.summary().additional_properties; + let by = if sum.contains_key("orca.run") { + "orca" + } else { + "other" + }; + let parts = sum + .get("orca.partitions") + .filter(|v| !v.is_empty()) + .cloned() + .unwrap_or_else(|| "-".into()); + (s.snapshot_id(), by.to_string(), parts) + } + }; + println!("{ds} {snap} {by} {parts}"); + } + 0 +} + +/// `keys`: the keys dataset's snapshot, then each distinct value of its +/// column, sorted. +pub(crate) fn keys(p: &Built) -> i32 { + let Some((ds, col)) = p.cfg.keys_dataset.clone() else { + return 2; + }; + let run = || -> Result<(i64, Vec), String> { + let cat = catalog()?; + let Some(t) = block(load(&cat, &ident(&ds)?))? else { + return Ok((-1, vec![])); + }; + let snap = current(t.metadata()); + if snap < 0 { + return Ok((-1, vec![])); + } + let batches: Vec = block(async { + t.scan() + .select([col.as_str()]) + .snapshot_id(snap) + .build() + .map_err(|e| e.to_string())? + .to_arrow() + .await + .map_err(|e| e.to_string())? + .try_collect() + .await + .map_err(|e| e.to_string()) + })?; + let mut keys = std::collections::BTreeSet::new(); + for b in &batches { + let c = b + .column_by_name(&col) + .and_then(|c| c.as_any().downcast_ref::()) + .ok_or_else(|| format!("morca: {ds}.{col} isn't a string column"))?; + for i in 0..c.len() { + if c.is_valid(i) { + keys.insert(c.value(i).to_string()); + } + } + } + Ok((snap, keys.into_iter().collect())) + }; + match run() { + Ok((snap, keys)) => { + println!("{snap}"); + for k in keys { + println!("{k}"); + } + 0 + } + Err(e) => { + eprintln!("{e}"); + 3 + } + } +} + +/// For tests and tools: appends `batches` to the dataset (made with +/// `schema` if there's none, unpartitioned) as a program that isn't a morca +/// pipeline would, with `summary` as its only extra summary properties. +/// Returns the snapshot. +pub fn append_as_other( + dataset: &str, + schema: Schema, + batches: Vec, + summary: HashMap, +) -> Result { + let cat = catalog()?; + let id = ident(dataset)?; + block(async { + let t = match load(&cat, &id).await? { + Some(t) => t, + None => { + let c = TableCreation::builder() + .name(id.name().to_string()) + .schema(schema) + .properties(table_defaults()) + .build(); + cat.create_table(id.namespace(), c) + .await + .map_err(|e| e.to_string())? + } + }; + let files = write_files(&t, &batches).await.map_err(|e| e.to_string())?; + let tx = Transaction::new(&t); + let tx = tx + .fast_append() + .add_data_files(files) + .set_snapshot_properties(summary) + .apply(tx) + .map_err(|e| e.to_string())?; + let t = tx.commit(&cat).await.map_err(|e| e.to_string())?; + Ok(current(t.metadata())) + }) +} + +/// A two-column schema (`id` long, `name` string), as the examples use. +pub fn id_name_schema() -> Schema { + use iceberg::spec::{NestedField, PrimitiveType, Type}; + Schema::builder() + .with_schema_id(0) + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::required(2, "name", Type::Primitive(PrimitiveType::String)).into(), + ]) + .build() + .expect("a valid schema") +} + +/// `n` rows of [`id_name_schema`], named `-`. +pub fn id_name_rows(prefix: &str, n: usize) -> Result { + use arrow_array::{ArrayRef, Int64Array}; + let schema = Arc::new(schema_to_arrow_schema(&id_name_schema()).map_err(|e| e.to_string())?); + let ids: ArrayRef = Arc::new(Int64Array::from_iter_values(0..n as i64)); + let names: ArrayRef = Arc::new(StringArray::from_iter_values( + (0..n).map(|i| format!("{prefix}-{i}")), + )); + RecordBatch::try_new(schema, vec![ids, names]).map_err(|e| e.to_string()) +} diff --git a/rust/morca-sdk/src/tables/mod.rs b/rust/morca-sdk/src/tables/mod.rs new file mode 100644 index 00000000..5b8c2197 --- /dev/null +++ b/rust/morca-sdk/src/tables/mod.rs @@ -0,0 +1,37 @@ +//! Tables (docs/pipeline-protocol.md, "Tables"), with the `tables` feature: +//! the warehouse `MORCA_WAREHOUSE` as morca's filesystem catalog (dataset +//! `ns.table` at `/ns/table`, a local directory or +//! `s3://bucket/prefix` with the `AWS_*` variables), read and committed with +//! iceberg-rust 0.11.0 on arrow-rs 58. +//! +//! - A read scans the snapshot this attempt already read, else the one the +//! step's first attempt read (pinned), else the current one, and is +//! recorded in `orca.reads`. +//! - A commit is fenced, carries the `orca.*` summary keys, and is recorded +//! in `orca.commits`; before writing anything, `find_commit` looks for the +//! commit an earlier attempt of the run already made, and records that +//! instead. +//! - Only appends: iceberg-rust 0.11.0 has no overwrite or delete +//! transaction, so a step can't declare `replaces=` yet. +//! +//! Without the feature a pipeline declares no datasets, `sense` prints +//! nothing and `keys` can't be asked for. + +#[cfg(feature = "tables")] +mod catalog; +#[cfg(feature = "tables")] +mod imp; + +#[cfg(feature = "tables")] +pub use imp::*; + +#[cfg(not(feature = "tables"))] +pub(crate) fn sense(_p: &crate::pipeline::Built) -> i32 { + 0 +} + +#[cfg(not(feature = "tables"))] +pub(crate) fn keys(_p: &crate::pipeline::Built) -> i32 { + eprintln!("morca: keys needs morca-sdk's `tables` feature"); + 3 +} diff --git a/tests/conformance.py b/tests/conformance.py index d390b6f3..21ffb1bc 100755 --- a/tests/conformance.py +++ b/tests/conformance.py @@ -525,8 +525,14 @@ def stable(): "the same DAG reports the same registry hash", stable) def morca_version(): + if "morca" not in d: + # A binary not built with morca-mojo (a Rust one, #521) says + # which protocol it speaks instead (Versions). + need("protocol" in d, "neither a `morca` key nor a `protocol`: " + "register would check nothing") + return need(isinstance(d.get("morca"), str) and d["morca"], - "no `morca` key: register would check nothing") + "`morca` is empty: register would check nothing") if not b.morca: raise Skip("no morca binary to compare with") code, out, _ = subprocess_run([b.morca, "version"]) @@ -534,7 +540,9 @@ def morca_version(): need(d["morca"] == want, f"`morca` is {d['morca']}, the scheduler is {want}") self.check("describe.morca", "versions", - "describe's `morca` is the morca the binary targets", morca_version) + "describe's `morca` is the morca the binary targets, or a binary " + "not built with morca-mojo leaves it out and says `protocol`", + morca_version) def protocol(): need("protocol" in d, "describe names no protocol version") diff --git a/tests/conformance/clean-rust.json b/tests/conformance/clean-rust.json new file mode 100644 index 00000000..cda793f1 --- /dev/null +++ b/tests/conformance/clean-rust.json @@ -0,0 +1,13 @@ +{ + "pipeline": "clean_rust", + "partitions": "monthly", + "keys": ["2024-03", "2024-04"], + "steps": [ + {"name": "clean", "kind": "mapped", "reads": ["raw.events"], "writes": ["lake.clean"]} + ], + "sensor": true, + "warehouse": true, + "setup": [["build/cargo/release/examples/source", "{warehouse}", "2024-03"]], + "slow_env": {"MORCA_CLEAN_SLEEP": "3"}, + "commit_input": ["build/cargo/release/examples/source", "{warehouse}"] +} diff --git a/tests/conformance/toy-rust.json b/tests/conformance/toy-rust.json new file mode 100644 index 00000000..a17a071f --- /dev/null +++ b/tests/conformance/toy-rust.json @@ -0,0 +1,11 @@ +{ + "pipeline": "toy_rust", + "partitions": "monthly", + "keys": ["2024-01", "2024-02", "2024-03"], + "steps": [ + {"name": "days", "kind": "mapped"}, + {"name": "measure", "kind": "mapped"}, + {"name": "total", "kind": "gather"} + ], + "slow_env": {"MORCA_TOY_SLEEP": "3"} +} diff --git a/tests/rust_sdk.sh b/tests/rust_sdk.sh new file mode 100755 index 00000000..2ce51b82 --- /dev/null +++ b/tests/rust_sdk.sh @@ -0,0 +1,180 @@ +#!/bin/sh +# morca-sdk (#521): whole pipelines in Rust, with no Mojo toolchain +# (docs/rust-pipelines.md): +# +# - cargo test: the crate's own tests (keys, the builder's checks, describe +# and the registry hash, the crash summary), with and without tables; +# - examples/toy-rust, examples/toy.mojo rebuilt in Rust, links no Mojo +# runtime; +# - the conformance suite (tests/conformance.py) passes for the Rust toy and +# for the Rust sensor with tables (rust/morca-sdk/examples/clean.rs, the +# twin of tests/sensors/clean.mojo), over SQLite and Postgres; +# - side by side: the Mojo toy and the Rust toy registered with one morca +# scheduler (the Rust one placed on the control host), fired by the same +# schedule, then a backfill each, every run succeeding; `morca status` +# and the UI's pages list both, and the control run's driver log was kept +# by the Rust binary's `keep-log`. +# +# Run in the rust environment, which has cargo and Mojo (for the scheduler +# and the Mojo toy), with a throwaway Postgres: +# pixi run -e rust test-rust-sdk +set -eu +cd "$(dirname "$0")/.." +. "$PWD/tests/port.sh" +mkdir -p build/rust-sdk +export CARGO_TARGET_DIR=$PWD/build/cargo +# On Linux, cargo links with conda-forge's compiler, against its older +# glibc. C code a crate builds (aws-lc-sys, under the tables feature) must +# use that compiler too: the system's gcc compiles against the system's +# glibc headers and leaves symbols (__isoc23_sscanf) the link can't find. +if [ "$(uname)" = Linux ] && [ -z "${CC:-}" ]; then + for c in "${CONDA_TOOLCHAIN_HOST:-none}-cc" x86_64-conda-linux-gnu-cc aarch64-conda-linux-gnu-cc; do + if command -v "$c" > /dev/null 2>&1; then + CC=$c + CXX=${c%-cc}-c++ + export CC CXX + break + fi + done +fi +log=build/rust-sdk/test.log +: > "$log" +ui= +sched= +fail() { + echo "FAIL: $*" + tail -40 "$log" + [ -z "$ui" ] || kill "$ui" 2> /dev/null || true + [ -z "$sched" ] || kill "$sched" 2> /dev/null || true + exit 1 +} +q() { psql "$MORCA_DATABASE_URL" -qAtc "$1"; } + +# The scheduler and the Mojo toy, while cargo builds. +# shellcheck disable=SC2086 +scripts/mojo-build.sh -I src $DEP_INC bin/scheduler.mojo -o build/morca >> "$log" 2>&1 & +mojo_morca=$! +# shellcheck disable=SC2086 +scripts/mojo-build.sh -I src $DEP_INC examples/toy.mojo -o build/rust-sdk/toy >> "$log" 2>&1 & +mojo_toy=$! + +cargo test --manifest-path rust/morca-sdk/Cargo.toml >> "$log" 2>&1 || fail "cargo test" +cargo test --manifest-path rust/morca-sdk/Cargo.toml --features tables >> "$log" 2>&1 \ + || fail "cargo test --features tables" +echo "ok the crate's tests, with and without tables" +cargo build --release --manifest-path examples/toy-rust/Cargo.toml >> "$log" 2>&1 || fail "build the Rust toy" +cargo build --release --features tables --examples --manifest-path rust/morca-sdk/Cargo.toml >> "$log" 2>&1 \ + || fail "build the table examples" +rtoy=$CARGO_TARGET_DIR/release/toy_rust +case "$(uname)" in +Darwin) libs=$(otool -L "$rtoy") ;; +*) libs=$(ldd "$rtoy" || true) ;; +esac +if echo "$libs" | grep -Eqi 'KGEN|mojo|liblake'; then + fail "the Rust toy links Mojo: $libs" +fi +echo "ok the Rust toy builds with cargo alone ($(wc -c < "$rtoy" | tr -d ' ') bytes) and links no Mojo runtime" + +wait "$mojo_morca" || fail "build morca" +wait "$mojo_toy" || fail "build the Mojo toy" + +python3 tests/conformance.py "$rtoy" tests/conformance/toy-rust.json || fail "conformance: the Rust toy" +python3 tests/conformance.py "$CARGO_TARGET_DIR/release/examples/clean" tests/conformance/clean-rust.json \ + || fail "conformance: the Rust sensor with tables" + +# Side by side under one scheduler, in a database of their own. +db=morca_side_by_side +q "DROP DATABASE IF EXISTS $db" > /dev/null +q "CREATE DATABASE $db" > /dev/null +MORCA_DATABASE_URL=$(echo "$MORCA_DATABASE_URL" | sed "s|///morca?|///$db?|") +export MORCA_DATABASE_URL +export MORCA_LOGS=$PWD/build/rust-sdk/logs MORCA_STALE_SECONDS=2 +rm -rf "$MORCA_LOGS" +# ` sample` runs the morca on the path. +PATH=$PWD/build:$PATH +export PATH +settle() { + i=0 + while [ "$(q "SELECT count(*) FROM orca.runs WHERE status IN ('queued', 'running')")" -gt 0 ]; do + i=$((i + 1)) + [ "$i" -lt 600 ] || fail "runs didn't finish: $(q "SELECT id, pipeline, status FROM orca.runs")" + sleep 0.1 + done +} +# The scheduler's clock stands at 03:00 on 2024-02-02, when both fire. +export MORCA_NOW=2024-02-02T03:00Z +MORCA_NOW=2024-01-15T00:00Z ./build/morca register ./build/rust-sdk/toy >> "$log" 2>&1 || fail "register the Mojo toy" +MORCA_NOW=2024-01-15T00:00Z ./build/morca register --placement control "$rtoy" >> "$log" 2>&1 \ + || fail "register the Rust toy" +./build/morca scheduler >> "$log" 2>&1 & +sched=$! +i=0 +until [ "$(q "SELECT count(*) FROM orca.runs WHERE trigger = 'schedule'")" -eq 2 ]; do + i=$((i + 1)); [ "$i" -lt 300 ] || fail "the schedule didn't fire both"; sleep 0.1 +done +settle +for p in toy toy_rust; do + [ "$(q "SELECT count(*) FROM orca.runs WHERE pipeline = '$p' AND partitions = '2024-01' AND status = 'succeeded' AND trigger = 'schedule'")" -eq 1 ] \ + || fail "$p's scheduled run for 2024-01 didn't succeed: $(q "SELECT id, status, error FROM orca.runs WHERE pipeline = '$p'")" +done +for p in toy toy_rust; do + ./build/morca backfill "$p" 2024-02..2024-04 >> "$log" 2>&1 || fail "backfill $p" +done +sleep 0.5 +settle +for p in toy toy_rust; do + n=$(q "SELECT count(*) FROM orca.runs WHERE pipeline = '$p' AND status = 'succeeded'") + [ "$n" -eq 4 ] || fail "$p has $n succeeded runs, not 4: $(q "SELECT id, partitions, status, error FROM orca.runs WHERE pipeline = '$p'")" +done +first=$(q "SELECT min(id) FROM orca.runs WHERE pipeline = 'toy_rust'") +# The scheduler hands a control run's output to the binary's keep-log once +# its process has ended. +i=0 +until [ -s "$MORCA_LOGS/toy_rust/$first/driver.log" ]; do + i=$((i + 1)); [ "$i" -lt 100 ] || fail "no driver log for control run $first (keep-log)"; sleep 0.1 +done +kill "$sched" 2> /dev/null || true +wait "$sched" 2> /dev/null || true +[ "$(q "SELECT count(*) FROM orca.runs WHERE status <> 'succeeded'")" -eq 0 ] || fail "a run didn't succeed" +[ "$(q "SELECT DISTINCT placement FROM orca.runs WHERE pipeline = 'toy_rust'")" = control ] \ + || fail "the Rust toy's runs weren't placed on the control host" +grep -q "^--- start " "$MORCA_LOGS/toy_rust/$first/driver.log" || fail "the driver log has no start line" +[ -s "$MORCA_LOGS/toy_rust/$first/measure/2024-01/1.log" ] || fail "no attempt log for run $first" +echo "--- morca status" +./build/morca status | tee -a "$log" +./build/morca status | grep -q "^toy_rust .*last run [0-9]* succeeded" || fail "status doesn't list the Rust toy" +./build/morca status | grep -q "^toy .*last run [0-9]* succeeded" || fail "status doesn't list the Mojo toy" +echo "--- runs" +q "SELECT id, pipeline, partitions, trigger, status, placement FROM orca.runs ORDER BY id" | tee -a "$log" + +free_port port +base="http://127.0.0.1:$port" +MORCA_UI_ADDR="127.0.0.1:$port" ./build/morca ui >> "$log" 2>&1 & +ui=$! +i=0 +until curl -fs "$base/" > /dev/null 2>&1; do + i=$((i + 1)); [ "$i" -lt 100 ] || fail "morca ui didn't start"; sleep 0.1 +done +for p in toy toy_rust; do + curl -fsS -H 'Accept: application/json' "$base/pipelines/$p" > "build/rust-sdk/$p.json" || fail "GET /pipelines/$p" +done +for p in toy toy_rust; do + python3 -c ' +import json, sys +d = json.load(open(sys.argv[1])) +assert [s["name"] for s in d["steps"]] == ["days", "measure", "total"], d["steps"] +assert d["pipeline"]["last_run"]["status"] == "succeeded", d["pipeline"]["last_run"] +assert len(d["runs"]) == 4 and all(r["status"] == "succeeded" for r in d["runs"]), d["runs"] +print(" /pipelines/%s: steps %s, %d runs succeeded, placement %s" % (d["pipeline"]["name"], + ",".join(s["name"] for s in d["steps"]), len(d["runs"]), d["pipeline"]["placement"])) +' "build/rust-sdk/$p.json" || fail "/pipelines/$p's JSON" +done +curl -fsS "$base/pipelines/toy_rust" | grep -q "measure" || fail "the Rust toy's page doesn't show its steps" +curl -fsS "$base/runs/$first" | grep -q "succeeded" || fail "run $first's page" +curl -fsS "$base/runs" > build/rust-sdk/runs.html || fail "GET /runs" +for p in toy toy_rust; do + grep -q "/pipelines/$p\"" build/rust-sdk/runs.html || fail "/runs doesn't list $p's runs" +done +kill "$ui" 2> /dev/null || true +ui= +echo "ok the Mojo toy and the Rust toy ran side by side under one scheduler (status, /runs and their pages)"