Fix the high findings from the e2e test review

.github/workflows/ci.yml

@@ -174,9 +174,19 @@ jobs:
       - name: Set up Rust toolchain
         uses: ./.github/actions/setup-rust
       - name: Run the e2e suite
+        # Its own timeout, below the job's, so a hang still leaves time to upload the job dirs.
+        timeout-minutes: 30
         run: scripts/e2e.sh
         env:
           E2E_REQUIRE_NFT: "1"
+          E2E_JOBS_DIR: ${{ runner.temp }}/e2e-jobs
+      - name: Upload the e2e job dirs
+        if: failure() || cancelled()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: e2e-jobs
+          path: ${{ runner.temp }}/e2e-jobs
+          retention-days: 7
 
   hbench-action-smoke-test:
     name: hbench-action smoke test

scripts/e2e.sh

@@ -5,7 +5,10 @@ set -euo pipefail
 cd "$(dirname "$0")/.."
 cargo build -q
 BIN=target/debug/hbench
-JOBS=$(mktemp -d)
+# E2E_JOBS_DIR keeps the job dirs somewhere known (CI uploads it when a run fails).
+JOBS=${E2E_JOBS_DIR:-$(mktemp -d)}
+mkdir -p "$JOBS"
+trap 'rc=$?; [ $rc = 0 ] || echo "e2e failed (exit $rc); job dirs: $JOBS" >&2' EXIT
 
 $BIN run -p tests/fixtures -a oracle -n 4 -k 2 --jobs-dir "$JOBS" --job-name fixtures
 
@@ -23,20 +26,31 @@ expect = {
     "no-network":      ({"reward": 1.0}, None),
     "compose-sidecar": ({"reward": 1.0}, None),
     "prebuilt":        ({"reward": 1.0}, None),
+    "planted-reward":  ({"reward": 1.0}, None),
 }
-seen = {}
-for r in job.glob("*/result.json"):
+def explain(trial):
+    """What a failed trial left behind: its exception, the end of its log, and the verifier's output."""
+    out = []
+    for name in ("exception.txt", "trial.log", "verifier/test-stdout.txt"):
+        p = trial / name
+        if p.exists():
+            out.append(f"--- {name} (tail)\n" + "\n".join(p.read_text(errors="replace").splitlines()[-30:]))
+    return "\n".join(out)
+seen, bad = {}, []
+for r in sorted(job.glob("*/result.json")):
     d = json.loads(r.read_text())
     task = d["task_name"]
     rewards = (d["verifier_result"] or {}).get("rewards")
     exc = (d["exception_info"] or {}).get("exception_type")
     want_r, want_e = expect[task]
-    assert (rewards, exc) == (want_r, want_e), f"{task}: got {rewards} {exc}, want {want_r} {want_e}"
+    if (rewards, exc) != (want_r, want_e):
+        bad.append(f"{r.parent.name}: got {rewards} {exc}, want {want_r} {want_e}\n{explain(r.parent)}")
     seen[task] = seen.get(task, 0) + 1
+assert not bad, "\n\n".join(bad)
 assert seen == {t: 2 for t in expect}, seen
 res = json.loads((job / "result.json").read_text())
 s = res["stats"]
-assert res["finished_at"] and s["n_completed_trials"] == 20 and s["n_errored_trials"] == 8, s
+assert res["finished_at"] and s["n_completed_trials"] == 22 and s["n_errored_trials"] == 8, s
 ev = s["evals"]["oracle__fixtures"]
 assert ev["exception_stats"].keys() == {"AgentTimeoutError", "RewardFileNotFoundError"}, ev
 print("fixture assertions OK")
@@ -60,9 +74,14 @@ EOF
 # Docker Desktop included; E2E_REQUIRE_NFT insists on it) and under the proxy-only fallback.
 if docker info --format '{{.OperatingSystem}}' | grep -q 'Docker Desktop'; then model_host=host.docker.internal; else model_host=172.17.0.1; fi
 model_port=18080
-python3 -m http.server "$model_port" --bind 0.0.0.0 --directory "$JOBS" >/dev/null 2>&1 &
+# An empty dir: the stand-in must not serve the job dirs to whoever can reach the port.
+mkdir -p "$JOBS/model-standin"
+python3 -m http.server "$model_port" --bind 0.0.0.0 --directory "$JOBS/model-standin" >"$JOBS/model-standin.log" 2>&1 &
 model_pid=$!
-trap 'kill $model_pid 2>/dev/null' EXIT
+trap 'rc=$?; kill $model_pid 2>/dev/null; [ $rc = 0 ] || echo "e2e failed (exit $rc); job dirs: $JOBS" >&2' EXIT
+# A stand-in that never came up would read as blocked, not as a pass, but fail here with the reason.
+for _ in $(seq 50); do curl -fs -o /dev/null "http://127.0.0.1:$model_port/" && break; sleep 0.1; done
+curl -fs -o /dev/null "http://127.0.0.1:$model_port/" || { cat "$JOBS/model-standin.log"; exit 1; }
 default_method=${E2E_REQUIRE_NFT:+nft+proxy}
 runs="false true true:proxy"
 for run in $runs; do

src/diff.rs

@@ -123,25 +123,31 @@ fn without_task(v: &Value) -> Value {
     v
 }
 
+/// The name a target's results will carry (`[task].name`, else the dir), which is what sources match on.
+/// The lock's name is always the dir, so it can't be compared with `result.task_name`.
+fn target_task_name(c: &TrialConfig, lock: &Value) -> String {
+    crate::task::Task::load(&crate::registry::local_path(&c.task))
+        .map(|t| t.name())
+        .unwrap_or_else(|_| lock["task"]["name"].as_str().unwrap_or_default().to_string())
+}
+
 pub fn compare(targets: &[TrialConfig], sources: &[Source]) -> Result<Vec<TrialDiff>> {
     let locks: Vec<Value> = targets.iter().map(trial_lock).collect();
-    for (c, l) in targets.iter().zip(&locks) {
-        let name = l["task"]["name"].as_str().unwrap_or_default();
+    let names: Vec<String> = targets.iter().zip(&locks).map(|(c, l)| target_task_name(c, l)).collect();
+    for (l, name) in locks.iter().zip(&names) {
         if let Some(v) = l["task"]["version"].as_str() {
             semver(v)?;
-            for s in sources.iter().filter(|s| s.result.task_name == name && s.version() == Some(v)) {
+            for s in sources.iter().filter(|s| s.result.task_name == *name && s.version() == Some(v)) {
                 if s.lock["task"]["digest"] != l["task"]["digest"] {
                     bail!("Task '{name}' has version {v} in both source and target locks, but the digests differ.");
                 }
             }
         }
-        let _ = c;
     }
     let mut consumed: HashSet<usize> = HashSet::new();
     let mut regradable: BTreeMap<PathBuf, Option<String>> = BTreeMap::new();
     let mut out = vec![];
-    for (c, l) in targets.iter().zip(&locks) {
-        let task_name = l["task"]["name"].as_str().unwrap_or_default().to_string();
+    for ((c, l), task_name) in targets.iter().zip(&locks).zip(names) {
         let rerun = |reason: String, downgraded: bool| TrialDiff {
             action: Action::Rerun,
             target_trial_name: c.trial_name.clone(),
@@ -271,4 +277,25 @@ mod tests {
         assert_ne!(without_task(&lock("0.2.5", None)), without_task(&lock("0.2.7", Some("0.2"))));
         assert_ne!(without_task(&lock("0.2.5", None)), without_task(&lock("0.2.6", None)));
     }
+
+    #[test]
+    fn a_named_task_is_reused_from_its_earlier_trial() {
+        // hello-world's [task].name is "hbench/hello-world", not its dir name.
+        let task = crate::config::TaskRef { path: Some("examples/tasks/hello-world".into()), ..Default::default() };
+        let target = TrialConfig::new(task, "hello-world__new".into(), "jobs/new".into());
+        let lock = trial_lock(&target);
+        let result: TrialResult = serde_json::from_value(serde_json::json!({
+            "id": uuid::Uuid::new_v4(), "task_name": "hbench/hello-world", "trial_name": "hello-world__old",
+            "trial_uri": "", "task_id": null, "source": null, "task_checksum": "", "config": target,
+            "agent_info": {"name": "oracle", "version": "1", "model_info": null},
+            "agent_result": null, "verifier_result": {"rewards": {"reward": 1}}, "exception_info": null,
+            "started_at": "2026-01-01T00:00:00Z", "finished_at": "2026-01-01T00:01:00Z",
+            "environment_setup": null, "agent_setup": null, "agent_execution": null, "verifier": null,
+        }))
+        .unwrap();
+        let sources = [Source { dir: "jobs/old/hello-world__old".into(), result, lock }];
+        let d = compare(&[target], &sources).unwrap();
+        assert_eq!(d[0].action, Action::Reuse, "{}", d[0].reason);
+        assert_eq!(d[0].task_name, "hbench/hello-world");
+    }
 }