Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changes/528-conformance-suite.minor.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Conformance suite**: `pixi run conformance <binary> <manifest>` checks a pipeline binary against docs/pipeline-protocol.md. (#528)
3 changes: 2 additions & 1 deletion docs/pipeline-protocol.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ deals with it only through four things: the binary's command line, its
environment, the run store (Postgres, SQLite, or the UI's worker API) and the
Iceberg tables. This page writes that contract down, so that a second
implementation, such as a pipeline binary written in Rust (#521), can be
built from it. The conformance suite (#519) tests against it.
built from it. The conformance suite (`tests/conformance.py`, #519) tests
against it: `pixi run conformance <binary> <manifest>`.

Today the Mojo side of the contract is `morca_main` (`src/morca/_cli.mojo`),
`Pipeline` (`src/morca/pipeline.mojo`), `Ctx` (`src/morca/ctx.mojo`) and the
Expand Down
14 changes: 12 additions & 2 deletions pixi.toml
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,13 @@ test-scheduler = { cmd = "scripts/with-postgres.sh tests/scheduler.sh" }
test-chaos = { cmd = "scripts/with-postgres.sh tests/chaos.sh" }
# The same over a SQLite file (morca.sqlite_store), with no server.
test-chaos-sqlite = { cmd = "MORCA_DATABASE_URL=sqlite:build/chaos.db tests/chaos.sh" }
# The conformance suite (#519): any pipeline binary against
# docs/pipeline-protocol.md, over SQLite and a throwaway Postgres, given a
# manifest of what the pipeline is (tests/conformance.py):
# pixi run conformance <binary> <manifest>
conformance = { cmd = "scripts/with-postgres.sh python3 tests/conformance.py" }
# The suite on toy, daily and fanout, and tests/chaos.sh in its mode.
test-conformance = { cmd = "scripts/with-postgres.sh tests/conformance.sh examples" }
test-ui = { cmd = "scripts/with-postgres.sh tests/ui.sh" }
test-actions = { cmd = "scripts/with-postgres.sh tests/actions.sh" }
# The fleet page (morca.fleet): samples kept in Postgres, hosts, cost, charts.
Expand Down Expand Up @@ -283,7 +290,7 @@ test-kind = { cmd = "tests/kind.sh" }
build-shared = { cmd = "scripts/build-shared.sh" }
# CI runs these three parts on runners of their own; `test` is all of them.
# The executor, stores, crashes, what runs use and the data a run makes:
test-executor-core = { depends-on = ["build-shared", "test-sample", "test-steps", "test-cron", "test-warehouse", "test-cli", "test-hetzner", "test-compile-fail", "test-store", "test-formal-pins", "test-state", "test-formal-migrate", "test-formal-fence", "test-crash", "test-tasks", "test-logs", "test-usage", "test-resources", "test-recommend", "test-cgroups", "test-aws", "test-source", "test-highlight", "test-step-cards", "test-graph", "test-locate", "test-timeline", "test-datasets", "test-db-blip", "test-chaos", "test-chaos-sqlite"] }
test-executor-core = { depends-on = ["build-shared", "test-sample", "test-steps", "test-cron", "test-warehouse", "test-cli", "test-hetzner", "test-compile-fail", "test-store", "test-formal-pins", "test-state", "test-formal-migrate", "test-formal-fence", "test-crash", "test-tasks", "test-logs", "test-usage", "test-resources", "test-recommend", "test-cgroups", "test-aws", "test-source", "test-highlight", "test-step-cards", "test-graph", "test-locate", "test-timeline", "test-datasets", "test-db-blip", "test-chaos", "test-chaos-sqlite", "test-conformance"] }
# Workers, pools, placement and the clouds they run on:
test-executor-workers = { depends-on = ["build-shared", "test-ec2", "test-autoscale", "test-no-workers", "test-pool", "test-hcloud", "test-pool-scale", "test-pool-scale-ec2", "test-pool-image", "test-run-cluster", "test-secrets", "test-placement", "test-grants", "test-pressure", "test-lost-workers"] }
test-executor = { depends-on = ["test-executor-core", "test-executor-workers"] }
Expand Down Expand Up @@ -438,6 +445,9 @@ test-commits = { cmd = "tests/concurrent_commits.sh", depends-on = ["build-libla
# Snapshot expiry: morca expire, morca_maintain and the scheduler's sweep.
test-expire = { cmd = "scripts/with-postgres.sh tests/expire.sh", depends-on = ["build-liblake"] }
test-sensors = { cmd = "scripts/with-postgres.sh tests/sensors.sh", depends-on = ["build-liblake"] }
# The conformance suite (#519) on a sensor with tables: commits, reads,
# pinned snapshots and a retry after a commit, then `sense`.
test-conformance-tables = { cmd = "scripts/with-postgres.sh tests/conformance.sh tables", depends-on = ["build-liblake"] }
# keys_per_run for sensors and fills (#240), over tests/sensors.
test-keys-per-run = { cmd = "scripts/with-postgres.sh tests/keys_per_run.sh", depends-on = ["build-liblake"] }
# A snapshot sensed while a run of its partition is live (#139).
Expand All @@ -456,7 +466,7 @@ test-keyed-names = { cmd = "scripts/with-postgres.sh tests/keyed_names.sh", depe
test-worker = { cmd = "scripts/with-postgres.sh tests/worker.sh", depends-on = ["build-liblake"] }
# CI runs these three groups on runners of their own; test-data is all three.
test-data-tables = { depends-on = ["build-shared", "test-tables", "test-tables-source", "test-lake", "test-lake-batch", "test-lake-batch-source", "test-append-files", "test-replace", "test-replace-concurrent", "test-formal-retry", "test-run-token", "test-chaos-tables", "test-lineage", "test-dataview", "test-commits", "test-expire", "test-taxi"] }
test-data-sensors = { depends-on = ["build-shared", "test-sensors", "test-keys-per-run", "test-sensed-behind-live", "test-lineage-s3", "test-sensors-s3", "test-sample-s3", "test-worker"] }
test-data-sensors = { depends-on = ["build-shared", "test-sensors", "test-conformance-tables", "test-keys-per-run", "test-sensed-behind-live", "test-lineage-s3", "test-sensors-s3", "test-sample-s3", "test-worker"] }
test-data-keyed = { depends-on = ["build-shared", "test-keyed", "test-keyed-backfills", "test-keyed-sensor", "test-keyed-names"] }
test-data = { depends-on = ["test-data-tables", "test-data-sensors", "test-data-keyed"] }
taxi = { cmd = "scripts/mojo-build.sh -I src -I examples/taxi $DEP_INC examples/taxi/taxi.mojo -o build/taxi", depends-on = ["build-liblake"] }
Expand Down
45 changes: 37 additions & 8 deletions tests/chaos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,14 @@
# --allow-new-code, which resumes it to the end (what tests/kill.sh checked
# before it was folded in here, #447).
#
# With CHAOS_BIN, the conformance suite's mode (tests/conformance.py,
# docs/pipeline-protocol.md): CHAOS_BIN is a prebuilt binary of the chaos
# pipeline, in any language, with its knobs (CHAOS_NAP, CHAOS_FLAKY,
# CHAOS_SEED and a `TOTAL <n>` log line), and every run, the reference one
# too, is started as the runners start one: a run row as the scheduler
# makes it (`conformance.py new-run`) and `resume`, never `run`. The keys are
# CHAOS_KEYS, comma-separated (the twelve months of 2024).
#
# Needs MORCA_DATABASE_URL and MORCA_PGDATA (pixi run test-chaos starts a
# throwaway server), or MORCA_DATABASE_URL=sqlite:<path>, a file it starts
# afresh (pixi run test-chaos-sqlite). CHAOS_TIMEOUT (default 120) seconds
Expand All @@ -36,7 +44,9 @@ src=${CHAOS_SRC:-tests/chaos_pipeline.mojo}
# By its full path, so the check for processes left behind (pgrep) sees
# only this checkout's, not another job's on the same machine.
bin=$PWD/build/$(basename "$src" .mojo)
if [ "$src" = tests/chaos_pipeline.mojo ]; then
if [ -n "${CHAOS_BIN:-}" ]; then
bin=$(cd "$(dirname "$CHAOS_BIN")" && pwd)/$(basename "$CHAOS_BIN")
elif [ "$src" = tests/chaos_pipeline.mojo ]; then
scripts/pipelines.sh chaos_pipeline
else
# shellcheck disable=SC2086
Expand All @@ -48,6 +58,7 @@ export MORCA_WORKERS=2
export CHAOS_NAP=${CHAOS_NAP:-0.05}
export CHAOS_FLAKY=${CHAOS_FLAKY:-30}
months=${CHAOS_MONTHS:-2024-01..2024-12}
keys=${CHAOS_KEYS:-2024-01,2024-02,2024-03,2024-04,2024-05,2024-06,2024-07,2024-08,2024-09,2024-10,2024-11,2024-12}
case $MORCA_DATABASE_URL in
sqlite:*)
# The same queries, on Postgres's tables without the morca schema.
Expand All @@ -66,6 +77,10 @@ sqlite:*)
;;
esac

# A new run of the pipeline over its months, left for `resume` as the
# scheduler leaves one (conformance mode only).
new_run() { python3 tests/conformance.py new-run "$bin" "$keys" "$@"; }

# A fresh warehouse for a pipeline with tables.
fresh_warehouse() {
CHAOS_WH=$(pwd)/build/chaos-wh-$1-$$
Expand All @@ -76,7 +91,12 @@ fresh_warehouse() {

# What an uninterrupted run gives: its total and its number of steps.
[ "${CHAOS_CHECK:-0}" = 1 ] && fresh_warehouse ref
ref=$(env -u MORCA_DATABASE_URL CHAOS_FLAKY=0 CHAOS_NAP= "$bin" run --partitions "$months" 2>&1)
if [ -n "${CHAOS_BIN:-}" ]; then
ref_run=$(new_run) || { echo "FAIL: no reference run"; exit 1; }
ref=$(CHAOS_FLAKY=0 CHAOS_NAP= "$bin" resume "$ref_run" 2>&1)
else
ref=$(env -u MORCA_DATABASE_URL CHAOS_FLAKY=0 CHAOS_NAP= "$bin" run --partitions "$months" 2>&1)
fi
want=$(echo "$ref" | sed -n 's/.*TOTAL //p')
steps=$(echo "$ref" | grep ' succeeded$' | grep -vc '^run ')
[ -n "$want" ] || { echo "FAIL: the reference run printed no total"; echo "$ref"; exit 1; }
Expand Down Expand Up @@ -116,9 +136,14 @@ for seed in ${CHAOS_SEEDS:-1 2 3}; do
fresh_warehouse "$seed" || fail "init failed"
fi
started=$SECONDS
"$bin" run --partitions "$months" >> "$log" 2>&1 &
pids=($!)
run=""
if [ -n "${CHAOS_BIN:-}" ]; then
run=$(new_run) || fail "no run"
"$bin" resume "$run" >> "$log" 2>&1 &
else
"$bin" run --partitions "$months" >> "$log" 2>&1 &
fi
pids=($!)
until [ -n "$run" ]; do
run=$(sed -n 's/^run \([0-9][0-9]*\)$/\1/p' "$log" | head -1)
[ $((SECONDS - started)) -lt 20 ] || fail "the run never started"
Expand Down Expand Up @@ -224,10 +249,14 @@ if [ "${CHAOS_CHECK:-0}" != 1 ]; then
tail -40 "$log"
exit 1
}
CHAOS_FLAKY=100 CHAOS_NAP= "$bin" run --partitions "$months" > "$log" 2>&1 && fail "a run whose flaky step always fails succeeded"
run=$(sed -n 's/^run \([0-9][0-9]*\)$/\1/p' "$log" | head -1)
[ -n "$run" ] || fail "the failing run never started"
q "UPDATE orca.runs SET registry_hash = '0xdead' WHERE id = $run" > /dev/null
if [ -n "${CHAOS_BIN:-}" ]; then
run=$(new_run --hash 0xdead) || fail "no run"
else
CHAOS_FLAKY=100 CHAOS_NAP= "$bin" run --partitions "$months" > "$log" 2>&1 && fail "a run whose flaky step always fails succeeded"
run=$(sed -n 's/^run \([0-9][0-9]*\)$/\1/p' "$log" | head -1)
[ -n "$run" ] || fail "the failing run never started"
q "UPDATE orca.runs SET registry_hash = '0xdead' WHERE id = $run" > /dev/null
fi
set +e
CHAOS_FLAKY=0 CHAOS_NAP= "$bin" resume "$run" > "$log" 2>&1
code=$?
Expand Down
Loading
Loading