From 693250d5fc9de2c2f77d018f74cb1b2c26bfdf6f Mon Sep 17 00:00:00 2001 From: rzonyx <77541804+rzonyx@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:44:45 +0700 Subject: [PATCH] CI: run the e2e suite's three parts in parallel The e2e job was the slowest in CI (about 9 minutes, against 2 for every other job), and a release waits for CI on the tagged commit, so it set the release's time. E2E_PART now runs one of three parts of scripts/e2e.sh, each about a third of the suite in the last run: runs (the fixture suite and the commands that read its job: jobs, traces, the viewer; ~220s), isolation (integrity, egress, secrets; ~90s) and cli (the rest of the CLI; ~220s). The ci.yml job is a matrix over the three, each uploading its own job dirs on a failure. Unset, the script runs all three as before. The macOS jobs stay as they are. Their runners queued for minutes on two of the last 14 CI runs, but building both macOS targets in one job would add about 5 minutes to every release to save that wait only sometimes. --- .github/workflows/ci.yml | 10 ++++++++-- docs/REFERENCE.md | 4 +++- scripts/e2e.sh | 26 +++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8277cb6..f112cef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,12 @@ jobs: e2e: # The integrity guarantees (stopping the agent at timeout, the hidden-test scrub, the egress allowlist) # only show against a real daemon. - name: E2E + name: E2E (${{ matrix.part }}) + strategy: + fail-fast: false + matrix: + # scripts/e2e.sh's parts, run in parallel; each is about a third of the suite. + part: [runs, isolation, cli] runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -178,6 +183,7 @@ jobs: timeout-minutes: 30 run: scripts/e2e.sh env: + E2E_PART: ${{ matrix.part }} E2E_REQUIRE_NFT: "1" E2E_JOBS_DIR: ${{ runner.temp }}/e2e-jobs # The registry the dataset commands push to and pull from; the same pin as the OCI tests above. @@ -186,7 +192,7 @@ jobs: if: failure() || cancelled() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: e2e-jobs + name: e2e-jobs-${{ matrix.part }} path: ${{ runner.temp }}/e2e-jobs retention-days: 7 diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 6aa848c..3f48c9c 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -452,11 +452,13 @@ npm run dev # hot reload; proxies /api to `hb `exec`, `analyze` and `check` with the nop agent, `tasks start-env`, `cache clean --dry`, and the viewer's API. Last, it checks for leaked containers. Not covered: `hbench update` (it would install a release over the binary under test) and the interactive paths (the `run` wizard, a real `trials handoff`, the `start-env` shell). + `E2E_PART=runs|isolation|cli` runs one part: the fixture suite and the commands that read its job; integrity, + egress and secrets; or the rest of the CLI. Unset, it runs all three. - `scripts/install-matrix.sh`: installs every installed agent into a real container (`install_only`). CI (`.github/workflows/ci.yml`) runs on GitHub-hosted runners: build + `cargo test` on Linux and macOS, `cargo fmt --check` and clippy, `cargo audit`, a check that the committed `viewer/dist` matches `viewer/src`, `scripts/e2e.sh` -against a real Docker daemon, and the example tasks through +against a real Docker daemon (its three parts in parallel jobs), and the example tasks through [node-q/hbench-action](https://github.com/node-q/hbench-action) with the latest release: every one must score 1 with the oracle and 0 with `nop`. On Linux it also runs the OCI tests against two local `registry:3` containers (`HBENCH_TEST_OCI_REGISTRY=localhost:5000 cargo test oci::tests`), the second taking basic auth with a password diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 1f6e331..ec60282 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -54,6 +54,13 @@ wait_for() { return 1 } +# part : whether this run includes that part. E2E_PART picks one, so CI runs the three in parallel: runs (the +# fixture suite and the commands that read its job), isolation (integrity, egress, secrets) and cli (the rest of the +# CLI). Unset, it runs all three. +case ${E2E_PART:-} in "" | runs | isolation | cli) ;; *) echo "E2E_PART must be runs, isolation or cli, not $E2E_PART" >&2; exit 2 ;; esac +part() { [ -z "${E2E_PART:-}" ] || [ "$E2E_PART" = "$1" ]; } + +if part runs; then $BIN run -p tests/fixtures -a oracle -n 4 -k 2 --jobs-dir "$JOBS" --job-name fixtures python3 - "$JOBS/fixtures" <<'EOF' @@ -99,7 +106,9 @@ ev = s["evals"]["oracle__fixtures"] assert ev["exception_stats"].keys() == {"AgentTimeoutError", "RewardFileNotFoundError"}, ev print("fixture assertions OK") EOF +fi +if part isolation; then # Verifier-only files datasets build into the agent's image are gone before a (non-oracle) agent starts. $BIN run -p tests/integrity -a nop --jobs-dir "$JOBS" --job-name integrity >/dev/null python3 - "$JOBS/integrity" <<'EOF' @@ -197,12 +206,14 @@ names = {p.name for p in files if b"[REDACTED]" in p.read_bytes()} assert {"oracle.txt", "trajectory-notes.txt", "leak.txt", "test-stdout.txt"} <= names, names print("secret assertions OK") EOF +fi # ------------------------------------------------------------------------------------------------------ # The rest of the CLI. Everything below this line writes under $CLI or into a named job dir under $JOBS. CLI=$JOBS/cli mkdir -p "$CLI" +if part cli; then # --- Version, completions, schemas, the agent list --------------------------------------------------- $BIN --version | check "hbench " $BIN completions bash | check "_hbench" @@ -299,7 +310,9 @@ $BIN tasks check -p "$CLI/migrated" | check "ok " $BIN run -p "$CLI/migrated/hello-legacy" -a oracle --jobs-dir "$JOBS" --job-name migrated >/dev/null rewards "$JOBS/migrated" | check "{'reward': 1.0} None" echo "authoring assertions OK" +fi +if part runs; then # --- Jobs: start, summary, verify, resume, regrade, upload ------------------------------------------ $BIN jobs start -p examples/tasks/hello-alpine -a nop --jobs-dir "$JOBS" --job-name start >/dev/null rewards "$JOBS/start" | check "hbench/hello-alpine {'reward': 0.0} None" @@ -369,7 +382,9 @@ expect_fail "(401" $BIN jobs upload "$JOBS/start" --url "$upload_url/ok" expect_fail "E2E_MISSING_TOKEN is not set" $BIN jobs upload "$JOBS/start" --url "$upload_url/ok" --token-env E2E_MISSING_TOKEN expect_fail "differ from its manifest.json" env E2E_UPLOAD_TOKEN=$upload_token $BIN jobs upload "$CLI/tampered" --url "$upload_url/ok" --token-env E2E_UPLOAD_TOKEN -H "X-E2E: yes" echo "jobs assertions OK" +fi +if part cli; then # --- Trials: start, handoff ------------------------------------------------------------------------- $BIN trials start -p examples/tasks/hello-world --trials-dir "$CLI/trials" --ae E2E_VAR=1 > "$CLI/trial.json" python3 -c ' @@ -381,7 +396,9 @@ expect_fail "does not support handoff" $BIN trials handoff "$trial" expect_fail "not a trial directory" $BIN trials handoff "$CLI" expect_fail "pass -p or -c " $BIN trials start echo "trials assertions OK" +fi +if part runs; then # --- Traces: an ATIF trajectory planted in a copy of a finished trial --------------------------------- cp -r "$JOBS/start" "$CLI/traces" python3 - "$CLI/traces" <<'EOF' @@ -408,7 +425,9 @@ assert rows[1]["conversations"][-1] == {"role": "assistant", "content": "Done."} expect_fail "no exportable trials" $BIN traces export -p "$CLI/traces" --filter success expect_fail "no exportable trials" $BIN traces export -p "$CLI/traces" --no-recursive echo "traces assertions OK" +fi +if part cli; then # --- Datasets: an OCI layout, a registry container, a registry.json over a git checkout ------------- D=$CLI/datasets mkdir -p "$D" @@ -570,7 +589,9 @@ $BIN tasks start-env -p examples/tasks/hello-world --non-interactive --no-all -- $BIN cache clean --dry | check "Docker images:" $BIN cache clean --dry --no-docker --no-cache-dir | check "Docker images: 0" echo "start-env and cache assertions OK" +fi +if part runs; then # --- The viewer: jobs mode on the job dirs, tasks mode on the examples, read-only off loopback ------ view() { # view : start the viewer, print its base URL. local log=$1 @@ -595,7 +616,9 @@ check "Read-only" < "$CLI/view-readonly.log" [ -d "$JOBS/start" ] expect_fail "does not exist" $BIN view "$CLI/nowhere" echo "viewer assertions OK" +fi +if part cli; then # hbench view follows a running trial: the agent's logs reach the trial dir before the agent exits. cp -R examples/tasks/hello-alpine "$JOBS/live-task" cat > "$JOBS/live-task/solution/solve.sh" <<'EOF' @@ -619,7 +642,8 @@ done wait $live_pid [ -n "$live_seen" ] || { echo "agent logs did not reach the host during the run"; exit 1; } echo "live sync OK" +fi left=$(docker ps -a --filter label=com.docker.compose.service=main --format '{{.Names}}' | grep -E '__[0-9a-f]{7}-main' || true) [ -z "$left" ] || { echo "leaked containers: $left"; exit 1; } -echo "E2E OK ($JOBS)" +echo "E2E OK (${E2E_PART:-all parts}, $JOBS)"