Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions changelog.d/1101-handoff-run-epoch.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Fixed
The `scripts/dsh-superagent-run.sh` handoff no longer reports an earlier run that
is still going. The launch epoch is taken with the `status.json` snapshot, and a
changed file is only reported when its `started` is numeric and no older than
that epoch, so a handoff into a reused store names its own run — or
`found: false` — instead of the earlier run, whose `run_id` the page's read-back
would then have polled (#1101).
9 changes: 6 additions & 3 deletions docs/dsh-super-agent.md
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,12 @@ exits 0 within seconds. Post the run id, the round and the run directory on the
Ticket and end the turn; nothing holds the slot.

A store is reused for every Ticket, so it already holds earlier runs. The
wrapper only reports a `status.json` that appears or changes after it launches,
so a handoff into a used store reports `found: false` until its own run writes
one, instead of naming a finished run from last week. The tmux session carries a
wrapper only reports a `status.json` that appears after it launches, or one that
changes with a `started` no older than the launch, so a handoff into a used store
reports `found: false` until its own run writes one. A file whose `started`
predates the launch is never reported: that is a run this handoff did not start
— possibly still going and rewriting its own file every round — and naming it
would send the read-back to the wrong run. The tmux session carries a
per-launch suffix for the same reason: two handoffs into stores both called
`runs` get two sessions, not one shared, refused name. One live handoff per
store: two launched into one store inside the same wait window both read that
Expand Down
34 changes: 31 additions & 3 deletions scripts/dsh-superagent-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,11 @@ QUOTED="$(printf '%q ' "${CMD[@]}")"
SNAP="$STORE/.handoff-$$-$RANDOM.json"
trap 'rm -f "$SNAP"' EXIT
python3 - "$STORE" "$SNAP" <<'PY'
import glob, json, os, sys
import glob, json, os, sys, time
store, out = sys.argv[1:3]
snap = {}
# The launch epoch rides in the snapshot: `started` is written once at launch
# and never reset, so a changed file can be checked against it (#1101).
snap = {"epoch": time.time()}
paths = [os.path.join(store, "status.json")] + sorted(
glob.glob(os.path.join(store, "*", "status.json")))
for path in paths:
Expand Down Expand Up @@ -83,7 +85,22 @@ python3 - "$STORE" "$WAIT" "$DRIVER" "$SESSION" "$LOG" "$QUOTED" "$SNAP" <<'PY'
import glob, json, os, sys, time
store, wait, driver, session, log, command, snap_path = sys.argv[1:8]
before = json.load(open(snap_path, encoding="utf-8"))
epoch = float(before.pop("epoch"))
deadline, found = time.time() + float(wait), None


def started_at(path):
"""A run's launch time, or None when the file does not carry a numeric one."""
try:
with open(path, encoding="utf-8") as fh:
started = json.load(fh).get("started")
except (OSError, ValueError):
return None
if isinstance(started, bool) or not isinstance(started, (int, float)):
return None
return float(started)


while time.time() < deadline:
paths = [p for p in [os.path.join(store, "status.json")]
+ sorted(glob.glob(os.path.join(store, "*", "status.json")))
Expand All @@ -95,8 +112,19 @@ while time.time() < deadline:
except OSError:
continue
seen = before.get(path)
if seen is None or [st.st_mtime, st.st_size] != seen:
if seen is None:
# Not there when the snapshot was taken: this launch made it.
fresh.append(path)
continue
if [st.st_mtime, st.st_size] == seen:
continue
# A reused store can hold a run that is still going, and that run
# rewrites its own status.json every round (#1101). Only a file whose
# `started` is no older than this launch is this handoff's.
started = started_at(path)
if started is None or started < epoch:
continue
fresh.append(path)
if fresh:
found = max(fresh, key=os.path.getmtime)
break
Expand Down
78 changes: 78 additions & 0 deletions tests/test_dsh_superagent_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,84 @@ def test_a_reused_store_does_not_report_an_earlier_run(tmp_path):
assert os.path.realpath(report["run_dir"]) == os.path.realpath(str(store)), report


LIVE_EARLIER = '''
import json, os, sys, time
# A run that is still going: runstore rewrites its status.json every round, so
# its mtime and size move in the handoff window exactly like a new run's file.
path = os.path.join(sys.argv[1], "status.json")
rounds = 0
while True:
rounds += 1
with open(path, "w", encoding="utf-8") as fh:
json.dump({"run_id": "old-run", "started": 1000.0, "round": rounds,
"state": "running"}, fh)
time.sleep(0.2)
'''

LATE_NEW = '''
import json, os, sys, time
store = sys.argv[1]
time.sleep(1.2)
run_dir = os.path.join(store, "new-run")
os.makedirs(run_dir, exist_ok=True)
tmp = os.path.join(run_dir, "status.json.tmp")
with open(tmp, "w", encoding="utf-8") as fh:
json.dump({"run_id": "new-run", "started": time.time(), "round": 1,
"state": "running"}, fh)
os.replace(tmp, os.path.join(run_dir, "status.json"))
'''


@pytest.mark.skipif(os.environ.get("AGENTDESCENT_LONG_RUN_SMOKE") != "1",
reason="set AGENTDESCENT_LONG_RUN_SMOKE=1 to run the detached smoke")
def test_a_live_earlier_run_is_never_reported_as_this_handoff(tmp_path):
# A reused store can hold a run that is *still going*, and runstore
# rewrites that run's status.json every round (#1101). Its file then moves
# in the window exactly like the new run's, which the mtime/size snapshot
# cannot tell apart -- the first poll adopted the earlier run and the page's
# read-back polled it. `started` is written once at launch and never reset,
# so the launch epoch taken with the snapshot is what separates the two.
store = tmp_path / "runs"
old = store / "old-run"
old.mkdir(parents=True)
(old / "status.json").write_text(
json.dumps({"run_id": "old-run", "started": 1000.0, "round": 1,
"state": "running"}), encoding="utf-8")
live = tmp_path / "live_earlier.py"
live.write_text(LIVE_EARLIER, encoding="utf-8")
late = tmp_path / "late_new.py"
late.write_text(LATE_NEW, encoding="utf-8")
ticking = subprocess.Popen([sys.executable, str(live), str(old)])
try:
# Nothing this launch started: the ticking earlier run is not a handoff,
# so the report is the same gap as an empty store.
handed = subprocess.run(
["bash", str(WRAPPER), "--store", str(store), "--", "sleep", "1"],
cwd=REPO, capture_output=True, text=True, timeout=60,
env=dict(os.environ, DSH_RUN_HANDOFF_WAIT="1"))
assert handed.returncode == 0, handed.stderr
report = json.loads((store / "waiting.json").read_text(encoding="utf-8"))
assert report["found"] is False, report

# This launch's own run appears at +1.2s while the earlier one keeps
# ticking: the report names the new run and the read-back follows it.
handed = subprocess.run(
["bash", str(WRAPPER), "--store", str(store), "--",
sys.executable, str(late), str(store)],
cwd=REPO, capture_output=True, text=True, timeout=60,
env=dict(os.environ, DSH_RUN_HANDOFF_WAIT="3"))
assert handed.returncode == 0, handed.stderr
report = json.loads((store / "waiting.json").read_text(encoding="utf-8"))
assert report["run_id"] == "new-run", report
assert Path(report["run_dir"]).name == "new-run", report
followed = json.loads((Path(report["run_dir"]) / "status.json")
.read_text(encoding="utf-8"))
assert followed["run_id"] == "new-run", followed
finally:
ticking.terminate()
ticking.wait(timeout=10)


@pytest.mark.skipif(os.environ.get("AGENTDESCENT_LONG_RUN_SMOKE") != "1",
reason="set AGENTDESCENT_LONG_RUN_SMOKE=1 to run the detached smoke")
def test_two_handoffs_into_stores_called_runs_do_not_take_one_session(tmp_path):
Expand Down
Loading