diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8277cb6..f112cef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,12 @@ jobs: e2e: # The integrity guarantees (stopping the agent at timeout, the hidden-test scrub, the egress allowlist) # only show against a real daemon. - name: E2E + name: E2E (${{ matrix.part }}) + strategy: + fail-fast: false + matrix: + # scripts/e2e.sh's parts, run in parallel; each is about a third of the suite. + part: [runs, isolation, cli] runs-on: ubuntu-24.04 timeout-minutes: 45 steps: @@ -178,6 +183,7 @@ jobs: timeout-minutes: 30 run: scripts/e2e.sh env: + E2E_PART: ${{ matrix.part }} E2E_REQUIRE_NFT: "1" E2E_JOBS_DIR: ${{ runner.temp }}/e2e-jobs # The registry the dataset commands push to and pull from; the same pin as the OCI tests above. @@ -186,7 +192,7 @@ jobs: if: failure() || cancelled() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: e2e-jobs + name: e2e-jobs-${{ matrix.part }} path: ${{ runner.temp }}/e2e-jobs retention-days: 7 diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 6aa848c..3f48c9c 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -452,11 +452,13 @@ npm run dev # hot reload; proxies /api to `hb `exec`, `analyze` and `check` with the nop agent, `tasks start-env`, `cache clean --dry`, and the viewer's API. Last, it checks for leaked containers. Not covered: `hbench update` (it would install a release over the binary under test) and the interactive paths (the `run` wizard, a real `trials handoff`, the `start-env` shell). + `E2E_PART=runs|isolation|cli` runs one part: the fixture suite and the commands that read its job; integrity, + egress and secrets; or the rest of the CLI. Unset, it runs all three. - `scripts/install-matrix.sh`: installs every installed agent into a real container (`install_only`). CI (`.github/workflows/ci.yml`) runs on GitHub-hosted runners: build + `cargo test` on Linux and macOS, `cargo fmt --check` and clippy, `cargo audit`, a check that the committed `viewer/dist` matches `viewer/src`, `scripts/e2e.sh` -against a real Docker daemon, and the example tasks through +against a real Docker daemon (its three parts in parallel jobs), and the example tasks through [node-q/hbench-action](https://github.com/node-q/hbench-action) with the latest release: every one must score 1 with the oracle and 0 with `nop`. On Linux it also runs the OCI tests against two local `registry:3` containers (`HBENCH_TEST_OCI_REGISTRY=localhost:5000 cargo test oci::tests`), the second taking basic auth with a password diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 1f6e331..ec60282 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -54,6 +54,13 @@ wait_for() { return 1 } +# part : whether this run includes that part. E2E_PART picks one, so CI runs the three in parallel: runs (the +# fixture suite and the commands that read its job), isolation (integrity, egress, secrets) and cli (the rest of the +# CLI). Unset, it runs all three. +case ${E2E_PART:-} in "" | runs | isolation | cli) ;; *) echo "E2E_PART must be runs, isolation or cli, not $E2E_PART" >&2; exit 2 ;; esac +part() { [ -z "${E2E_PART:-}" ] || [ "$E2E_PART" = "$1" ]; } + +if part runs; then $BIN run -p tests/fixtures -a oracle -n 4 -k 2 --jobs-dir "$JOBS" --job-name fixtures python3 - "$JOBS/fixtures" <<'EOF' @@ -99,7 +106,9 @@ ev = s["evals"]["oracle__fixtures"] assert ev["exception_stats"].keys() == {"AgentTimeoutError", "RewardFileNotFoundError"}, ev print("fixture assertions OK") EOF +fi +if part isolation; then # Verifier-only files datasets build into the agent's image are gone before a (non-oracle) agent starts. $BIN run -p tests/integrity -a nop --jobs-dir "$JOBS" --job-name integrity >/dev/null python3 - "$JOBS/integrity" <<'EOF' @@ -197,12 +206,14 @@ names = {p.name for p in files if b"[REDACTED]" in p.read_bytes()} assert {"oracle.txt", "trajectory-notes.txt", "leak.txt", "test-stdout.txt"} <= names, names print("secret assertions OK") EOF +fi # ------------------------------------------------------------------------------------------------------ # The rest of the CLI. Everything below this line writes under $CLI or into a named job dir under $JOBS. CLI=$JOBS/cli mkdir -p "$CLI" +if part cli; then # --- Version, completions, schemas, the agent list --------------------------------------------------- $BIN --version | check "hbench " $BIN completions bash | check "_hbench" @@ -299,7 +310,9 @@ $BIN tasks check -p "$CLI/migrated" | check "ok " $BIN run -p "$CLI/migrated/hello-legacy" -a oracle --jobs-dir "$JOBS" --job-name migrated >/dev/null rewards "$JOBS/migrated" | check "{'reward': 1.0} None" echo "authoring assertions OK" +fi +if part runs; then # --- Jobs: start, summary, verify, resume, regrade, upload ------------------------------------------ $BIN jobs start -p examples/tasks/hello-alpine -a nop --jobs-dir "$JOBS" --job-name start >/dev/null rewards "$JOBS/start" | check "hbench/hello-alpine {'reward': 0.0} None" @@ -369,7 +382,9 @@ expect_fail "(401" $BIN jobs upload "$JOBS/start" --url "$upload_url/ok" expect_fail "E2E_MISSING_TOKEN is not set" $BIN jobs upload "$JOBS/start" --url "$upload_url/ok" --token-env E2E_MISSING_TOKEN expect_fail "differ from its manifest.json" env E2E_UPLOAD_TOKEN=$upload_token $BIN jobs upload "$CLI/tampered" --url "$upload_url/ok" --token-env E2E_UPLOAD_TOKEN -H "X-E2E: yes" echo "jobs assertions OK" +fi +if part cli; then # --- Trials: start, handoff ------------------------------------------------------------------------- $BIN trials start -p examples/tasks/hello-world --trials-dir "$CLI/trials" --ae E2E_VAR=1 > "$CLI/trial.json" python3 -c ' @@ -381,7 +396,9 @@ expect_fail "does not support handoff" $BIN trials handoff "$trial" expect_fail "not a trial directory" $BIN trials handoff "$CLI" expect_fail "pass -p or -c " $BIN trials start echo "trials assertions OK" +fi +if part runs; then # --- Traces: an ATIF trajectory planted in a copy of a finished trial --------------------------------- cp -r "$JOBS/start" "$CLI/traces" python3 - "$CLI/traces" <<'EOF' @@ -408,7 +425,9 @@ assert rows[1]["conversations"][-1] == {"role": "assistant", "content": "Done."} expect_fail "no exportable trials" $BIN traces export -p "$CLI/traces" --filter success expect_fail "no exportable trials" $BIN traces export -p "$CLI/traces" --no-recursive echo "traces assertions OK" +fi +if part cli; then # --- Datasets: an OCI layout, a registry container, a registry.json over a git checkout ------------- D=$CLI/datasets mkdir -p "$D" @@ -570,7 +589,9 @@ $BIN tasks start-env -p examples/tasks/hello-world --non-interactive --no-all -- $BIN cache clean --dry | check "Docker images:" $BIN cache clean --dry --no-docker --no-cache-dir | check "Docker images: 0" echo "start-env and cache assertions OK" +fi +if part runs; then # --- The viewer: jobs mode on the job dirs, tasks mode on the examples, read-only off loopback ------ view() { # view : start the viewer, print its base URL. local log=$1 @@ -595,7 +616,9 @@ check "Read-only" < "$CLI/view-readonly.log" [ -d "$JOBS/start" ] expect_fail "does not exist" $BIN view "$CLI/nowhere" echo "viewer assertions OK" +fi +if part cli; then # hbench view follows a running trial: the agent's logs reach the trial dir before the agent exits. cp -R examples/tasks/hello-alpine "$JOBS/live-task" cat > "$JOBS/live-task/solution/solve.sh" <<'EOF' @@ -619,7 +642,8 @@ done wait $live_pid [ -n "$live_seen" ] || { echo "agent logs did not reach the host during the run"; exit 1; } echo "live sync OK" +fi left=$(docker ps -a --filter label=com.docker.compose.service=main --format '{{.Names}}' | grep -E '__[0-9a-f]{7}-main' || true) [ -z "$left" ] || { echo "leaked containers: $left"; exit 1; } -echo "E2E OK ($JOBS)" +echo "E2E OK (${E2E_PART:-all parts}, $JOBS)"