From b736160ea59502e7c8c9fb0125abd89b15d2cdc1 Mon Sep 17 00:00:00 2001 From: HugoXK Date: Mon, 16 Nov 2026 05:11:27 +0000 Subject: [PATCH 1/3] fix(hosts): a child that never started is a failed handoff, not waiting The wrapper wrote `waiting.json` and exited 0 whatever the child did, so a child that refused to start read as a successful handoff: `status: waiting`, `found: false`, exit 0, with the real reason only in `worker.log` (#1096). `found: false` alone cannot tell "alive, no status yet" from "gone", so the end of the wait window is where liveness decides. tmux answers with `has-session -t =SESSION` (a session ends with its command); setsid answers with `kill(pid, 0)` on the pid the wrapper already started. When the child is gone and no fresh `status.json` appeared, the report says `status: failed`, `state: exited`, keeps the same ten fields with `log` naming `worker.log`, and the wrapper exits 2. A child still running at the window's end is unchanged: `waiting`, `found: false`, exit 0. Three rows describe children that outlive the window, so their children now do: the metacharacter row's marker child sleeps past it, and the two reused-store rows hand off `sleep 3` instead of `sleep 1`. No assertion changed -- a child that exits with no status is the failed handoff this commit adds. Report and page carry the new contract; one `Fixed` fragment. --- changelog.d/1096-handoff-failed-child.md | 7 +++ docs/dsh-super-agent.md | 8 ++++ scripts/dsh-superagent-run.sh | 54 +++++++++++++++++++++--- tests/test_dsh_superagent_run.py | 51 ++++++++++++++++++++-- 4 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 changelog.d/1096-handoff-failed-child.md diff --git a/changelog.d/1096-handoff-failed-child.md b/changelog.d/1096-handoff-failed-child.md new file mode 100644 index 00000000..04dc66d6 --- /dev/null +++ b/changelog.d/1096-handoff-failed-child.md @@ -0,0 +1,7 @@ +Fixed +The handoff wrapper no longer reports a run that never started as a successful +handoff: when the wait window closes with the child gone and no new +`status.json`, it writes `waiting.json` with `status: failed` and `state: +exited`, and exits nonzero with the child's own reason left in `worker.log`. A +child that is still running and has not written its status yet is unchanged: +`waiting`, `found: false`, exit 0. The page says both (#1096). diff --git a/docs/dsh-super-agent.md b/docs/dsh-super-agent.md index 3ffa281c..bdc6bad3 100644 --- a/docs/dsh-super-agent.md +++ b/docs/dsh-super-agent.md @@ -167,6 +167,14 @@ arguments rather than splicing them into a shell string, so a `--store` containing a quote, a space, `$` or `;` is used literally instead of being read as shell syntax (`#1109`). +A child that exits before writing a `status.json` handed nothing off, and the +wrapper says so: when the window closes with the run's session gone and no new +status file, `waiting.json` carries `status: failed` and `state: exited`, the +wrapper exits nonzero, and `log` names the `worker.log` where the child's own +reason sits (`#1096`). The window itself is unchanged for the case it exists +for — a run that is alive and has not written its status yet is still +`waiting`, with exit 0. + A store is reused for every Ticket, so it already holds earlier runs. The wrapper only reports a `status.json` that appears after it launches, or one that changes with a `started` no older than the launch, so a handoff into a used store diff --git a/scripts/dsh-superagent-run.sh b/scripts/dsh-superagent-run.sh index 3bddebe8..f79ae54b 100644 --- a/scripts/dsh-superagent-run.sh +++ b/scripts/dsh-superagent-run.sh @@ -16,7 +16,10 @@ # worker hands on. # # Exit 0 once the run is detached and DIR/waiting.json is written. It never -# waits for the run: a handoff that blocks is the failure this replaces. +# waits for the run: a handoff that blocks is the failure this replaces. Exit 2 +# when the child was already gone at the end of the wait window and wrote no +# status.json -- nothing was handed off, and DIR/worker.log holds its reason +# (#1096). set -euo pipefail WAIT="${DSH_RUN_HANDOFF_WAIT:-10}" # seconds to wait for the run's status.json @@ -78,11 +81,13 @@ with open(out, "w", encoding="utf-8") as fh: PY if command -v tmux >/dev/null 2>&1; then DRIVER=tmux + RUN_PID="" tmux new-session -d -s "$SESSION" -c "$PWD" "${LAUNCH[@]}" "${CMD[@]}" \ || die "tmux refused the session $SESSION" elif command -v setsid >/dev/null 2>&1; then DRIVER=setsid setsid "${LAUNCH[@]}" "${CMD[@]}" >/dev/null 2>&1 & + RUN_PID=$! else die "neither tmux nor setsid is available; this host cannot hand off a run" fi @@ -90,14 +95,37 @@ fi # The run owns its status.json (`//status.json`, or the store # itself when the caller owns the run directory). Absent is reported as such: # an invented run id is worse than a gap a Trigger can see. -python3 - "$STORE" "$WAIT" "$DRIVER" "$SESSION" "$LOG" "$QUOTED" "$SNAP" <<'PY' -import glob, json, os, sys, time -store, wait, driver, session, log, command, snap_path = sys.argv[1:8] +python3 - "$STORE" "$WAIT" "$DRIVER" "$SESSION" "$LOG" "$QUOTED" "$SNAP" "$RUN_PID" <<'PY' +import glob, json, os, subprocess, sys, time +store, wait, driver, session, log, command, snap_path, run_pid = sys.argv[1:9] before = json.load(open(snap_path, encoding="utf-8")) epoch = float(before.pop("epoch")) deadline, found = time.time() + float(wait), None +def still_running(): + """Whether the command this handoff started is alive at the window's end. + + `found: false` alone cannot tell "alive, no status yet" from "gone" (#1096). + The driver answers it: a tmux session ends with its command, and the setsid + pid is this shell's own child, so `kill(pid, 0)` says whether it is still + there. Unknown is reported as alive -- a false "failed" would be worse than + a missed one. + """ + if driver == "tmux": + return subprocess.run(["tmux", "has-session", "-t", "=" + session], + capture_output=True).returncode == 0 + if not run_pid: + return True + try: + os.kill(int(run_pid), 0) + except ProcessLookupError: + return False + except (PermissionError, ValueError): + return True + return True + + def started_at(path): """A run's launch time, or None when the file does not carry a numeric one.""" try: @@ -140,9 +168,16 @@ while time.time() < deadline: time.sleep(0.2) run_dir = os.path.dirname(found) if found else store status = json.load(open(found)) if found else {} -report = {"status": "waiting", "found": bool(found), +# Nothing this launch could report. If the child is gone too, nothing was handed +# off: the wrapper used to print `waiting` and exit 0 here, which is how a +# child that refused to start read as a successful handoff (#1096). The wait +# window exists for the other case -- alive, not written yet -- so only the +# driver's own liveness separates them. +expired = found is None and not still_running() +report = {"status": "failed" if expired else "waiting", "found": bool(found), "run_id": status.get("run_id", "") if found else "", - "round": status.get("round", 0), "state": status.get("state", "unknown"), + "round": status.get("round", 0), + "state": "exited" if expired else status.get("state", "unknown"), "run_dir": run_dir, "driver": driver, "session": session if driver == "tmux" else "", "log": log, "command": command.strip()} @@ -150,6 +185,11 @@ path = os.path.join(store, "waiting.json") with open(path, "w", encoding="utf-8") as fh: json.dump(report, fh, indent=2, sort_keys=True) fh.write("\n") -print(f"waiting {report['run_id'] or '(no run yet)'} round {report['round']} {run_dir}") +print(f"{report['status']} {report['run_id'] or '(no run yet)'} " + f"round {report['round']} {run_dir}") print(f"report {path}") +if expired: + print(f"error: the run exited before writing a status; its reason, if it " + f"wrote one, is in {log}", file=sys.stderr) + sys.exit(2) PY diff --git a/tests/test_dsh_superagent_run.py b/tests/test_dsh_superagent_run.py index 81d3ad81..3bb9b88f 100644 --- a/tests/test_dsh_superagent_run.py +++ b/tests/test_dsh_superagent_run.py @@ -172,10 +172,14 @@ def test_a_store_path_with_shell_metacharacters_hands_off_without_running_them( env["PATH"] = _path_without_tmux(tmp_path) # The child writes a marker, so "the run started" is measured rather than - # inferred from a log file the redirection alone would have created. + # inferred from a log file the redirection alone would have created, and it + # stays alive past the wait window: a child that has already exited with no + # status.json is now a failed handoff (#1096), and this row is about where + # the arguments went, not about when the child ends. marker = tmp_path / "ran.txt" child = [sys.executable, "-c", - "import pathlib, sys; pathlib.Path(sys.argv[1]).write_text('ran\\n')", + "import pathlib, sys, time; " + "pathlib.Path(sys.argv[1]).write_text('ran\\n'); time.sleep(2)", str(marker)] handed = subprocess.run( ["bash", str(WRAPPER), "--store", str(store), "--"] + child, @@ -198,6 +202,40 @@ def test_a_store_path_with_shell_metacharacters_hands_off_without_running_them( f"the run did not start through the handed-off log: {report}") +@pytest.mark.parametrize("driver", ["tmux", "setsid"]) +def test_a_child_that_exits_before_writing_a_status_is_a_failed_handoff( + tmp_path, driver): + # The wrapper promises to exit 0 once the run is detached and waiting.json + # is written, and it did that whatever the child did: a child that exited 2 + # before creating anything still produced `status: waiting`, `found: false` + # and exit 0, with its reason only in worker.log (#1096). `found: false` + # alone cannot tell "alive, no status yet" from "gone", so the end of the + # window is where liveness has to decide. + store = tmp_path / "gone" + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + if driver == "tmux": + if shutil.which("tmux") is None: + pytest.skip("no tmux on this host: the session row cannot run") + else: + if shutil.which("setsid") is None: + pytest.skip("no setsid on this host: the fallback row cannot run") + env["PATH"] = _path_without_tmux(tmp_path) + + child = [sys.executable, "-c", "import sys; sys.exit(2)"] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(store), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + report = json.loads((store / "waiting.json").read_text(encoding="utf-8")) + assert handed.returncode != 0, ( + "the child exited before writing a status and the wrapper still " + f"reported a handoff (exit {handed.returncode}): {report}") + assert report["found"] is False, report + assert report["status"] not in ("waiting", "running"), report + assert report["state"] not in ("waiting", "running"), report + assert report["log"] == str(store / "worker.log"), report + + @pytest.mark.skipif(os.environ.get("AGENTDESCENT_LONG_RUN_SMOKE") != "1", reason="set AGENTDESCENT_LONG_RUN_SMOKE=1 to run the detached smoke") def test_a_run_outlives_the_turn_that_started_it_and_leaves_no_orphan(tmp_path): @@ -273,8 +311,11 @@ def test_a_reused_store_does_not_report_an_earlier_run(tmp_path): (old / "status.json").write_text( json.dumps({"run_id": "old-run", "round": 7, "state": "finished"}), encoding="utf-8") + # `sleep 3` outlives the 1 s window: a child that has already exited with no + # status.json is a failed handoff (#1096), and this row needs one that is + # still going when the window closes. handed = subprocess.run( - ["bash", str(WRAPPER), "--store", str(store), "--", "sleep", "1"], + ["bash", str(WRAPPER), "--store", str(store), "--", "sleep", "3"], cwd=REPO, capture_output=True, text=True, timeout=60, env=dict(os.environ, DSH_RUN_HANDOFF_WAIT="1")) assert handed.returncode == 0, handed.stderr @@ -336,8 +377,10 @@ def test_a_live_earlier_run_is_never_reported_as_this_handoff(tmp_path): try: # Nothing this launch started: the ticking earlier run is not a handoff, # so the report is the same gap as an empty store. + # `sleep 3` outlives the 1 s window, for the same reason as the row + # above: the child must still be running when liveness is decided. handed = subprocess.run( - ["bash", str(WRAPPER), "--store", str(store), "--", "sleep", "1"], + ["bash", str(WRAPPER), "--store", str(store), "--", "sleep", "3"], cwd=REPO, capture_output=True, text=True, timeout=60, env=dict(os.environ, DSH_RUN_HANDOFF_WAIT="1")) assert handed.returncode == 0, handed.stderr From 151d2397f28554a41e6b01bd8ea7a607573ce10b Mon Sep 17 00:00:00 2001 From: HugoXK Date: Mon, 16 Nov 2026 08:13:47 +0000 Subject: [PATCH 2/3] fix(hosts): decide the handoff from a read taken at decision time (#1096) The review of #1117 found four ways the new `failed` verdict could be wrong: - the poll's last scan ran up to 200 ms before the decision and nothing re-read the store after the final sleep, so a status written in that interval was reported as a failed handoff while the file it asks about was there; - a caller-owned `/status.json` carries a `started` older than the launch by construction, and the epoch filter -- which exists for another run in a shared store -- rejected it forever, so a real handoff read as failed; - `tmux has-session` raising OSError killed the heredoc before `waiting.json` existed, so "cannot judge" produced no report at all; - the page and the script header still promised `found: false` for a run that writes its status into another store, which this rule reports as failed. The scan is now `fresh()`, called at decision time; a changed `/status.json` is accepted before the epoch filter; the probe treats OSError as alive and the report is written before the probe is asked; and the two-store text says what the wrapper now does. Four rows pin the cases, and the retimed children and every existing assertion are untouched. --- changelog.d/1096-handoff-failed-child.md | 10 +- docs/dsh-super-agent.md | 22 ++-- scripts/dsh-superagent-run.sh | 78 +++++++++---- tests/test_dsh_superagent_run.py | 140 +++++++++++++++++++++++ 4 files changed, 216 insertions(+), 34 deletions(-) diff --git a/changelog.d/1096-handoff-failed-child.md b/changelog.d/1096-handoff-failed-child.md index 04dc66d6..b0ba3fc2 100644 --- a/changelog.d/1096-handoff-failed-child.md +++ b/changelog.d/1096-handoff-failed-child.md @@ -1,7 +1,9 @@ Fixed The handoff wrapper no longer reports a run that never started as a successful handoff: when the wait window closes with the child gone and no new -`status.json`, it writes `waiting.json` with `status: failed` and `state: -exited`, and exits nonzero with the child's own reason left in `worker.log`. A -child that is still running and has not written its status yet is unchanged: -`waiting`, `found: false`, exit 0. The page says both (#1096). +`status.json` where the wrapper can see it, it writes `waiting.json` with +`status: failed` and `state: exited`, and exits nonzero with the child's own +reason left in `worker.log`. A child that is still running and has not written +its status yet is unchanged: `waiting`, `found: false`, exit 0. A store the +caller owns as the run directory is still reported from its own rewritten +`status.json`. The page says all three (#1096). diff --git a/docs/dsh-super-agent.md b/docs/dsh-super-agent.md index bdc6bad3..7691253a 100644 --- a/docs/dsh-super-agent.md +++ b/docs/dsh-super-agent.md @@ -152,9 +152,11 @@ the pair is what makes the run stop itself and write why. Two stores, one variable: the wrapper's `--store` is where the report goes, and the run's store is the CLI's own `--store`, so pass the same value on both. The -wrapper cannot see the child's store — give it only to the wrapper and `evolve` -writes to the default store, the wrapper waits out its deadline and reports -`found: false`, and the read-back below cannot find the run. +wrapper only sees the store it was given: give it only to the wrapper and +`evolve` writes to the default store, so the wrapper waits out its deadline, +finds no status of its own where it can look, and reports `status: failed` with +exit 2 — the child did start, but its status sits in a store this handoff cannot +read, and the read-back below cannot find the run. It starts the command in a detached `tmux` session (`setsid` when tmux is absent), waits for the run's own `status.json`, writes `waiting.json` beside it @@ -167,12 +169,14 @@ arguments rather than splicing them into a shell string, so a `--store` containing a quote, a space, `$` or `;` is used literally instead of being read as shell syntax (`#1109`). -A child that exits before writing a `status.json` handed nothing off, and the -wrapper says so: when the window closes with the run's session gone and no new -status file, `waiting.json` carries `status: failed` and `state: exited`, the -wrapper exits nonzero, and `log` names the `worker.log` where the child's own -reason sits (`#1096`). The window itself is unchanged for the case it exists -for — a run that is alive and has not written its status yet is still +A child that exits without leaving a `status.json` where the wrapper can see it +handed nothing off, and the wrapper says so: when the window closes with the +run's session gone and no new status file, `waiting.json` carries +`status: failed` and `state: exited`, the wrapper exits nonzero, and `log` names +the `worker.log` where the child's own reason sits (`#1096`). That includes a +run that wrote its status into a different store — the child started, this +handoff did not receive it. The window itself is unchanged for the case it +exists for — a run that is alive and has not written its status yet is still `waiting`, with exit 0. A store is reused for every Ticket, so it already holds earlier runs. The diff --git a/scripts/dsh-superagent-run.sh b/scripts/dsh-superagent-run.sh index f79ae54b..60249e0c 100644 --- a/scripts/dsh-superagent-run.sh +++ b/scripts/dsh-superagent-run.sh @@ -6,8 +6,9 @@ # dsh-superagent-run.sh --store DIR -- agentdescent --store DIR evolve spec.json --detach # # The wrapper's --store is where the report goes; the run's store is the CLI's -# own --store. Give both the same value, or the run lands in the default store -# and the report says found: false. +# own --store. Give both the same value: the wrapper decides "failed" from the +# store it was given, so a child that writes its status into a different store +# is reported as a failed handoff even when the run did start (#1096). # # tmux is preferred because dsh-superagent already depends on it and a session # is not in the worker's process group, so the kill that ends the worker's @@ -113,8 +114,14 @@ def still_running(): a missed one. """ if driver == "tmux": - return subprocess.run(["tmux", "has-session", "-t", "=" + session], - capture_output=True).returncode == 0 + try: + return subprocess.run(["tmux", "has-session", "-t", "=" + session], + capture_output=True).returncode == 0 + except OSError: + # The probe could not run at all (exec pressure, binary gone, + # permission): it did not answer, and "cannot judge" is alive. Letting + # this raise would kill the report before it is written (#1096 review). + return True if not run_pid: return True try: @@ -138,11 +145,18 @@ def started_at(path): return float(started) -while time.time() < deadline: +def fresh(): + """This launch's own status.json, read from the store now, or None. + + The scan is a function so the decision comes from a read taken at decision + time: the old loop exited on a scan up to 200 ms old, and a status.json + written during that last sleep was reported as a failed handoff (#1096 + review). + """ paths = [p for p in [os.path.join(store, "status.json")] + sorted(glob.glob(os.path.join(store, "*", "status.json"))) if os.path.isfile(p)] - fresh = [] + found_paths = [] for path in paths: try: st = os.stat(path) @@ -151,40 +165,62 @@ while time.time() < deadline: seen = before.get(path) if seen is None: # Not there when the snapshot was taken: this launch made it. - fresh.append(path) + found_paths.append(path) continue if [st.st_mtime, st.st_size] == seen: continue + if path == os.path.join(store, "status.json"): + # The store *is* the run directory when the caller owns it, so this + # file is this handoff's own run and its `started` predates the + # launch by construction. The epoch filter below is for another run + # in a shared store and must not reject it (#1096 review). + found_paths.append(path) + continue # A reused store can hold a run that is still going, and that run # rewrites its own status.json every round (#1101). Only a file whose # `started` is no older than this launch is this handoff's. started = started_at(path) if started is None or started < epoch: continue - fresh.append(path) - if fresh: - found = max(fresh, key=os.path.getmtime) + found_paths.append(path) + return max(found_paths, key=os.path.getmtime) if found_paths else None + + +while True: + found = fresh() + if found is not None or time.time() >= deadline: break time.sleep(0.2) run_dir = os.path.dirname(found) if found else store status = json.load(open(found)) if found else {} -# Nothing this launch could report. If the child is gone too, nothing was handed -# off: the wrapper used to print `waiting` and exit 0 here, which is how a -# child that refused to start read as a successful handoff (#1096). The wait -# window exists for the other case -- alive, not written yet -- so only the -# driver's own liveness separates them. -expired = found is None and not still_running() -report = {"status": "failed" if expired else "waiting", "found": bool(found), +report = {"status": "waiting", "found": bool(found), "run_id": status.get("run_id", "") if found else "", "round": status.get("round", 0), - "state": "exited" if expired else status.get("state", "unknown"), + "state": status.get("state", "unknown"), "run_dir": run_dir, "driver": driver, "session": session if driver == "tmux" else "", "log": log, "command": command.strip()} path = os.path.join(store, "waiting.json") -with open(path, "w", encoding="utf-8") as fh: - json.dump(report, fh, indent=2, sort_keys=True) - fh.write("\n") + + +def write_report(): + with open(path, "w", encoding="utf-8") as fh: + json.dump(report, fh, indent=2, sort_keys=True) + fh.write("\n") + + +# The report lands before the liveness question is asked: if the probe cannot +# run -- or the wrapper is killed while it waits -- a `waiting` report still +# exists, which is the direction #1096 calls for. Nothing this launch could +# report and the child gone means nothing was handed off; the wrapper used to +# print `waiting` and exit 0 there, which is how a child that refused to start +# read as a successful handoff (#1096). +write_report() +expired = found is None and not still_running() +if expired: + report["status"] = "failed" + report["state"] = "exited" + write_report() print(f"{report['status']} {report['run_id'] or '(no run yet)'} " f"round {report['round']} {run_dir}") print(f"report {path}") diff --git a/tests/test_dsh_superagent_run.py b/tests/test_dsh_superagent_run.py index 3bb9b88f..9f467698 100644 --- a/tests/test_dsh_superagent_run.py +++ b/tests/test_dsh_superagent_run.py @@ -236,6 +236,146 @@ def test_a_child_that_exits_before_writing_a_status_is_a_failed_handoff( assert report["log"] == str(store / "worker.log"), report +def _path_whose_tmux_vanishes(base: Path) -> str: + """A PATH whose only `tmux` answers `new-session` and then deletes itself. + + The wrapper checks `command -v tmux`, launches through it, and only after the + wait window asks `tmux has-session`. A tmux that answered the launch and can + no longer be run for the probe is the case where liveness cannot be judged at + all, so the probe must read as alive and the report must still exist (#1096 + review). + """ + real = shutil.which("tmux") + assert real is not None, "this host has no tmux for the launch half of the row" + shim = base / "vanishing" + shim.mkdir() + script = shim / "tmux" + script.write_text( + "#!/usr/bin/env bash\n" + f'"{real}" "$@"\n' + "rc=$?\n" + 'rm -f "$0"\n' + "exit $rc\n", + encoding="utf-8") + script.chmod(0o755) + return f"{shim}:{_path_without_tmux(base)}" + + +def test_a_status_written_in_the_last_poll_interval_is_reported(tmp_path): + # The old poll's last scan ran at deadline-0.2 s, then it slept and left, so a + # status written in that sleep was invisible: the store held the file and the + # report said failed. The decision has to come from a read at decision time + # (#1096 review). + store = tmp_path / "late" + if shutil.which("setsid") is None: + pytest.skip("no setsid on this host: the fallback row cannot run") + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + env["PATH"] = _path_without_tmux(tmp_path) + status = store / "live-run" / "status.json" + child = [sys.executable, "-c", + "import json, pathlib, sys, time; time.sleep(0.9); " + "p = pathlib.Path(sys.argv[1]); p.parent.mkdir(parents=True, " + "exist_ok=True); " + "p.write_text(json.dumps({'run_id': 'live-run', 'round': 1, " + "'state': 'running'}))", + str(status)] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(store), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + report = json.loads((store / "waiting.json").read_text(encoding="utf-8")) + assert handed.returncode == 0, ( + "a status written inside the last poll interval was reported as a " + f"failed handoff (exit {handed.returncode}): {report}\n{handed.stderr}") + assert report["found"] is True, report + assert report["run_id"] == "live-run", report + + +def test_a_caller_owned_store_reports_its_own_rewrite_as_the_handoff(tmp_path): + # When the caller owns the run directory, the store *is* the run, and the + # child's rewrite carries a `started` from before the launch by construction. + # The epoch filter is for another run in a shared store; applied here it + # turned a real handoff into failed + exit 2 (#1096 review). + store = tmp_path / "caller" + store.mkdir() + status = json.dumps({"started": 1000.0, "run_id": "caller-run", "round": 1, + "state": "running"}) + (store / "status.json").write_text(status, encoding="utf-8") + if shutil.which("setsid") is None: + pytest.skip("no setsid on this host: the fallback row cannot run") + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + env["PATH"] = _path_without_tmux(tmp_path) + child = [sys.executable, "-c", + "import json, pathlib, sys, time; time.sleep(0.3); " + "p = pathlib.Path(sys.argv[1]); " + "p.write_text(json.dumps({'started': 1000.0, " + "'run_id': 'caller-run', 'round': 4, 'state': 'finished'}))", + str(store / "status.json")] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(store), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + report = json.loads((store / "waiting.json").read_text(encoding="utf-8")) + assert handed.returncode == 0, ( + "the caller-owned store's own rewrite was reported as a failed handoff " + f"(exit {handed.returncode}): {report}\n{handed.stderr}") + assert report["found"] is True, report + assert report["run_id"] == "caller-run", report + assert report["round"] == 4, report + + +def test_a_probe_that_cannot_run_leaves_a_waiting_report(tmp_path): + # `tmux has-session` raising is "cannot judge", not "gone": the child here is + # still sleeping. An uncaught OSError killed the heredoc before the report was + # written, so the caller got exit 1 and no waiting.json at all (#1096 review). + store = tmp_path / "unjudgeable" + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + env["PATH"] = _path_whose_tmux_vanishes(tmp_path) + child = [sys.executable, "-c", "import time; time.sleep(2)"] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(store), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + assert (store / "waiting.json").is_file(), ( + f"no report was written when the probe could not run: {handed.stderr!r}") + report = json.loads((store / "waiting.json").read_text(encoding="utf-8")) + assert handed.returncode == 0, handed.stderr + assert report["status"] == "waiting", report + assert report["found"] is False, report + + +def test_a_run_written_to_a_different_store_is_a_failed_handoff(tmp_path): + # The page tells a worker to pass one store to both the wrapper and the CLI. + # A child that writes its status into a store the wrapper was not given has + # started, but this handoff cannot see it, and the wrapper now says failed + # (#1096 review). This row pins the documented behaviour. + reports = tmp_path / "reports" + other = tmp_path / "run-store" + if shutil.which("setsid") is None: + pytest.skip("no setsid on this host: the fallback row cannot run") + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + env["PATH"] = _path_without_tmux(tmp_path) + child = [sys.executable, "-c", + "import json, pathlib, sys; " + "p = pathlib.Path(sys.argv[1]) / 'run-1' / 'status.json'; " + "p.parent.mkdir(parents=True, exist_ok=True); " + "p.write_text(json.dumps({'run_id': 'run-1', 'round': 1, " + "'state': 'running'}))", + str(other)] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(reports), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + report = json.loads((reports / "waiting.json").read_text(encoding="utf-8")) + assert handed.returncode != 0, ( + "a run written to a store the wrapper cannot see was reported as a " + f"handoff (exit {handed.returncode}): {report}") + assert report["status"] == "failed", report + assert report["state"] == "exited", report + assert report["found"] is False, report + assert report["log"] == str(reports / "worker.log"), report + + @pytest.mark.skipif(os.environ.get("AGENTDESCENT_LONG_RUN_SMOKE") != "1", reason="set AGENTDESCENT_LONG_RUN_SMOKE=1 to run the detached smoke") def test_a_run_outlives_the_turn_that_started_it_and_leaves_no_orphan(tmp_path): From fdbd8afa7e675e2bc35dfaa2b3a79132b166faca Mon Sep 17 00:00:00 2001 From: HugoXK Date: Mon, 16 Nov 2026 08:40:06 +0000 Subject: [PATCH 3/3] fix(hosts): read the store after the liveness probe before failing a handoff (#1096) `expired` came from `fresh()` taken before `still_running()`, and the probe spawns a tmux client -- 10-30 ms, unbounded on a loaded box -- so a `status.json` written while it ran was on disk when the report said `found: false`, `status: failed`. A run that handed off just past the window was reported failed and retried. Re-read once after the probe and rebuild the report from the file; the provisional `waiting` report is replaced so a reader sees the handoff. The grace is the probe's own duration, not a wider window: a child that wrote nothing is still `failed`. The probe row also asserted tmux was installed where every sibling row skips, so `pytest -q` failed on a host without tmux; it now skips like them. The page says a caller-owned store must be the run this handoff starts, since its changed top-level `status.json` is taken as this launch's own. --- changelog.d/1096-handoff-failed-child.md | 4 +- docs/dsh-super-agent.md | 5 +- scripts/dsh-superagent-run.sh | 42 +++++++++--- tests/test_dsh_superagent_run.py | 82 +++++++++++++++++++++++- 4 files changed, 120 insertions(+), 13 deletions(-) diff --git a/changelog.d/1096-handoff-failed-child.md b/changelog.d/1096-handoff-failed-child.md index b0ba3fc2..77ef58e4 100644 --- a/changelog.d/1096-handoff-failed-child.md +++ b/changelog.d/1096-handoff-failed-child.md @@ -6,4 +6,6 @@ handoff: when the wait window closes with the child gone and no new reason left in `worker.log`. A child that is still running and has not written its status yet is unchanged: `waiting`, `found: false`, exit 0. A store the caller owns as the run directory is still reported from its own rewritten -`status.json`. The page says all three (#1096). +`status.json`. The verdict reads the store again after the liveness probe, so a +status written while the probe ran is still this handoff's. The page says all +three (#1096). diff --git a/docs/dsh-super-agent.md b/docs/dsh-super-agent.md index 7691253a..1faecb35 100644 --- a/docs/dsh-super-agent.md +++ b/docs/dsh-super-agent.md @@ -185,7 +185,10 @@ changes with a `started` no older than the launch, so a handoff into a used stor reports `found: false` until its own run writes one. A file whose `started` predates the launch is never reported: that is a run this handoff did not start — possibly still going and rewriting its own file every round — and naming it -would send the read-back to the wrong run. The tmux session carries a +would send the read-back to the wrong run. A caller-owned store, where the store +itself is the run directory, must be the run this handoff starts: the wrapper +takes its changed top-level `status.json` as this launch's own, so a second live +writer in that directory can be reported as this handoff. The tmux session carries a per-launch suffix for the same reason: two handoffs into stores both called `runs` get two sessions, not one shared, refused name. One live handoff per store: two launched into one store inside the same wait window both read that diff --git a/scripts/dsh-superagent-run.sh b/scripts/dsh-superagent-run.sh index 60249e0c..6b169044 100644 --- a/scripts/dsh-superagent-run.sh +++ b/scripts/dsh-superagent-run.sh @@ -186,20 +186,25 @@ def fresh(): return max(found_paths, key=os.path.getmtime) if found_paths else None +def make_report(found): + """The report for the status file this handoff is, or a bare `waiting`.""" + status = json.load(open(found)) if found else {} + return {"status": "waiting", "found": bool(found), + "run_id": status.get("run_id", "") if found else "", + "round": status.get("round", 0), + "state": status.get("state", "unknown"), + "run_dir": os.path.dirname(found) if found else store, + "driver": driver, + "session": session if driver == "tmux" else "", "log": log, + "command": command.strip()} + + while True: found = fresh() if found is not None or time.time() >= deadline: break time.sleep(0.2) -run_dir = os.path.dirname(found) if found else store -status = json.load(open(found)) if found else {} -report = {"status": "waiting", "found": bool(found), - "run_id": status.get("run_id", "") if found else "", - "round": status.get("round", 0), - "state": status.get("state", "unknown"), - "run_dir": run_dir, "driver": driver, - "session": session if driver == "tmux" else "", "log": log, - "command": command.strip()} +report = make_report(found) path = os.path.join(store, "waiting.json") @@ -217,12 +222,29 @@ def write_report(): # read as a successful handoff (#1096). write_report() expired = found is None and not still_running() +if expired: + # The probe spawns a tmux client -- 10-30 ms, unbounded on a loaded box -- + # and nothing reads the store while it runs, so a status.json written in + # that window is on disk when the verdict says it is not. Read once more at + # the moment of the decision and believe the file over the probe: a run + # that wrote and exited 20 ms after the window is a handoff, and calling it + # failed costs a duplicate run (#1096 review). This is the probe's window + # only; the deadline is not extended, so a child that wrote nothing is + # still `failed`. + late = fresh() + if late is not None: + found = late + report = make_report(found) + expired = False + # The provisional `waiting` report is already on disk and names no run; + # replace it so a reader sees the handoff this launch actually made. + write_report() if expired: report["status"] = "failed" report["state"] = "exited" write_report() print(f"{report['status']} {report['run_id'] or '(no run yet)'} " - f"round {report['round']} {run_dir}") + f"round {report['round']} {report['run_dir']}") print(f"report {path}") if expired: print(f"error: the run exited before writing a status; its reason, if it " diff --git a/tests/test_dsh_superagent_run.py b/tests/test_dsh_superagent_run.py index 9f467698..764a27e2 100644 --- a/tests/test_dsh_superagent_run.py +++ b/tests/test_dsh_superagent_run.py @@ -246,7 +246,8 @@ def _path_whose_tmux_vanishes(base: Path) -> str: review). """ real = shutil.which("tmux") - assert real is not None, "this host has no tmux for the launch half of the row" + if real is None: + pytest.skip("no tmux on this host: the launch half of the row cannot run") shim = base / "vanishing" shim.mkdir() script = shim / "tmux" @@ -291,6 +292,85 @@ def test_a_status_written_in_the_last_poll_interval_is_reported(tmp_path): assert report["run_id"] == "live-run", report +def _path_whose_tmux_answers_late(base: Path, delay: str) -> str: + """A PATH whose `tmux has-session` answers `delay` seconds after it is asked. + + The liveness probe spawns a tmux client and nothing reads the store while it + runs, so a status written in that window is on disk before the probe says + "gone". A real probe takes 10-30 ms and is unbounded on a loaded box; this + shim makes the window 0.6 s so the row can see it (#1096 review). + """ + real = shutil.which("tmux") + if real is None: + pytest.skip("no tmux on this host: the slow-probe row cannot run") + shim = base / "slow" + shim.mkdir(parents=True) + script = shim / "tmux" + script.write_text( + "#!/usr/bin/env bash\n" + f'if [ "$1" = "has-session" ]; then sleep {delay}; fi\n' + f'exec "{real}" "$@"\n', + encoding="utf-8") + script.chmod(0o755) + return f"{shim}:{_path_without_tmux(base)}" + + +def test_a_status_written_after_the_window_and_before_the_verdict_is_the_handoff( + tmp_path): + # The decision asked `fresh()`, then spawned the tmux probe, and did not look + # again: a status written while the probe ran was on disk when the report was + # written as failed. The verdict has to re-read the store after the probe, or + # a run that handed off 100 ms late is retried as if it never started + # (#1096 review). + base = tmp_path / "boundary" + store = base / "store" + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + env["PATH"] = _path_whose_tmux_answers_late(base, "0.6") + status = store / "late-run" / "status.json" + child = [sys.executable, "-c", + "import json, pathlib, sys, time; time.sleep(1.2); " + "p = pathlib.Path(sys.argv[1]); p.parent.mkdir(parents=True, " + "exist_ok=True); " + "p.write_text(json.dumps({'run_id': 'late-run', 'round': 2, " + "'state': 'running'}))", + str(status)] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(store), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + report = json.loads((store / "waiting.json").read_text(encoding="utf-8")) + assert status.is_file(), "the child did not write the status this row is about" + assert handed.returncode == 0, ( + "a status written after the window but before the verdict was reported " + f"as a failed handoff (exit {handed.returncode}): {report}\n{handed.stderr}") + assert report["found"] is True, report + assert report["run_id"] == "late-run", report + assert report["status"] == "waiting", report + + +def test_the_late_read_does_not_rescue_a_child_that_wrote_nothing(tmp_path): + # The grace is the probe's own duration, not a wider window: a child that is + # gone and wrote no status is still a failed handoff when the probe is slow, + # and the extra read must not turn it into a handoff (#1096 review). + base = tmp_path / "silent" + store = base / "store" + env = dict(os.environ, DSH_RUN_HANDOFF_WAIT="1") + env["PATH"] = _path_whose_tmux_answers_late(base, "0.6") + child = [sys.executable, "-c", "import sys; sys.exit(2)"] + handed = subprocess.run( + ["bash", str(WRAPPER), "--store", str(store), "--"] + child, + cwd=REPO, capture_output=True, text=True, timeout=60, env=env) + + report = json.loads((store / "waiting.json").read_text(encoding="utf-8")) + assert handed.returncode != 0, ( + "the child wrote nothing and the wrapper still reported a handoff " + f"(exit {handed.returncode}): {report}") + assert report["found"] is False, report + assert report["status"] == "failed", report + assert report["state"] == "exited", report + assert report["log"] == str(store / "worker.log"), report + + def test_a_caller_owned_store_reports_its_own_rewrite_as_the_handoff(tmp_path): # When the caller owns the run directory, the store *is* the run, and the # child's rewrite carries a `started` from before the launch by construction.