Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 44 additions & 11 deletions agentdescent/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -2381,10 +2381,19 @@ def _plan_agent_line(spec: Dict[str, Any]) -> Optional[str]:
return line


def _first_failed_probe_next(probe: Mapping[str, Any]) -> str:
"""The repair action from the first refused probe row (#1093)."""
for row in probe.get("rows") or []:
if row.get("state") == "failed" and row.get("next"):
return str(row["next"])
return "run `agentdescent doctor`, then re-run plan --probe"


def _print_plan(payload: Dict[str, Any], *, spec_path: str,
next_line: Optional[str] = None,
printed: Optional[set] = None,
probe_lines: Optional[Sequence[str]] = None) -> None:
probe_lines: Optional[Sequence[str]] = None,
then_line: Optional[str] = None) -> None:
"""Print the human `plan` summary; warnings and the held-out note go to stderr.

`payload` is exactly the object `--json` prints, so both surfaces carry the
Expand Down Expand Up @@ -2429,7 +2438,11 @@ def _print_plan(payload: Dict[str, Any], *, spec_path: str,
lines.append(f"note: (+{len(other_notes) - 3} more; --json lists every note)")
# The preflight (#1048) sits between the plan and the step it qualifies:
# `next:` stays the last line, so a reader who stops early still sees it.
# When the probe refused, that last line is the repair action, not evolve
# (#1093); `then:` names the command that re-runs the probe after the fix.
lines.extend(probe_lines or ())
if then_line:
lines.append(f"then: {then_line}")
lines.append(f"next: {next_line or f'agentdescent evolve {spec_path}'}")
print("\n".join(lines))

Expand Down Expand Up @@ -2497,24 +2510,35 @@ def cmd_plan(a: argparse.Namespace) -> int:
print(f"error: {payload.get('error', 'plan failed')}", file=sys.stderr)
return 2
probe = _probe_if_asked(a, spec) if payload.get("ok", True) else None
# Default next is evolve; a refused probe replaces it with the repair
# action so a reader (or a host following `next`) does not start a run
# that cannot work (#1093). `ok` stays about the plan/spec itself.
next_line = f"agentdescent evolve {a.spec}"
# Keep an ad-hoc `--algorithm` / `--param` override in the suggested
# command: it lives in memory, not in the spec file.
if getattr(a, "algorithm", None):
next_line += f" --algorithm {a.algorithm}"
for param in getattr(a, "param", None) or []:
next_line += f" --param {param}"
then_line = None
if probe is not None:
payload["probe"] = probe
payload["ready"] = bool(probe.get("ok", True))
if not payload["ready"]:
next_line = _first_failed_probe_next(probe)
payload["next"] = next_line
then_line = f"agentdescent plan {a.spec} --probe"
if a.json:
_out(payload, as_json=True)
else:
# The human form is the default; `--json` above is the machine form and
# stays exactly the object it always was (#919).
next_line = f"agentdescent evolve {a.spec}"
# Keep an ad-hoc `--algorithm` / `--param` override in the suggested
# command: it lives in memory, not in the spec file.
if getattr(a, "algorithm", None):
next_line += f" --algorithm {a.algorithm}"
for param in getattr(a, "param", None) or []:
next_line += f" --param {param}"
# stays exactly the object it always was (#919) when the probe is not
# asked. `ready` is added only when `--probe` ran.
from .probe import probe_lines as _probe_lines

_print_plan(payload, spec_path=a.spec, next_line=next_line,
probe_lines=(_probe_lines(probe) if probe is not None else None))
probe_lines=(_probe_lines(probe) if probe is not None else None),
then_line=then_line)
if not payload.get("ok", True):
return 2
# A refused probe is a config answer, not a spec error: the same non-zero
Expand Down Expand Up @@ -4558,11 +4582,19 @@ def _cmd_algorithms_run(a: argparse.Namespace) -> int:
report = obj.get("_report") or []
ok = bool(obj.get("ok", True))
probe = None
then_line = None
if getattr(a, "probe", False) and stage == "done" and ok:
# The same preflight `plan --probe` runs, on the spec this verb just
# wrote: one call, before the `evolve` it recommends (#1048).
probe = _probe_if_asked(a, load_spec(obj["spec_path"]))
obj["probe"] = probe
# `ok` stays about the written plan; `ready` is whether evolve can
# start. A refused probe also replaces `next` so a host does not
# follow it into a run that cannot work (#1093).
obj["ready"] = bool(probe.get("ok", True))
if not obj["ready"]:
obj["next"] = _first_failed_probe_next(probe)
then_line = f"agentdescent plan {obj['spec_path']} --probe"
if as_json:
# Exactly one JSON object on stdout for success and error (#674).
_out(_public(obj), as_json=True)
Expand Down Expand Up @@ -4603,7 +4635,8 @@ def _cmd_algorithms_run(a: argparse.Namespace) -> int:
from .probe import probe_lines as _probe_lines

_print_plan(payload, spec_path=out, next_line=obj["next"], printed=printed,
probe_lines=(_probe_lines(probe) if probe is not None else None))
probe_lines=(_probe_lines(probe) if probe is not None else None),
then_line=then_line)
calls = (payload.get("estimate") or {}).get("agent_calls_upper_bound")
if (getattr(a, "rounds", None) is None and getattr(a, "workers", None) is None
and isinstance(calls, (int, float)) and calls > QUICK_TRIAL_CALLS):
Expand Down
5 changes: 5 additions & 0 deletions changelog.d/1093-probe-failed-next.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fixed
A failed `plan --probe` or `algorithms run --probe` no longer ends with
`next: agentdescent evolve`. The last line (and the JSON `next`) is the
repair action, `--json` sets `ready: false` while leaving `ok` about the
plan itself, and a successful or omitted probe is unchanged (#1093).
6 changes: 4 additions & 2 deletions docs/algorithms.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@ agentdescent init PATH --algorithm NAME --param KEY=VALUE [--rounds N] [--worker
reflector before the run it recommends, so a key, endpoint, model or
subscription that cannot work is refused there instead of inside
`evolve` -- with the same `SubscriptionInactive` / `QuotaExhausted` /
`ProviderConfigError` answer and no run store entry. It is off by default:
plain `plan` still runs and spends nothing.
`ProviderConfigError` answer and no run store entry. A refused probe
ends with `next:` naming the repair (not `evolve`), and `--json`
sets `ready: false` while `ok` stays about the plan itself (#1093).
It is off by default: plain `plan` still runs and spends nothing.

Include cases the seed gets wrong: if the seed already scores
the target on held-out, the run stops at once with
Expand Down
5 changes: 4 additions & 1 deletion docs/plugin-quickstart.md
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,10 @@ Two habits worth keeping from the first run:
* `--probe` when the run is remote and the answer has to be known now: it sends
one minimal call per model-backed agent and reflector, so an expired plan, a
wrong model name or a dead endpoint is refused before a run store entry is
written. CLI agents are reported as `probe: … skipped (cli agent)` because one
written. If the probe fails, the last line is the repair action (not
`next: agentdescent evolve`); `then:` is `plan <spec> --probe` to re-test,
and `--json` sets `ready: false` while leaving `ok` about the plan itself.
CLI agents are reported as `probe: … skipped (cli agent)` because one
call would cost a whole session.
* `--budget 5 --usd-per-call 0.02` on the first real run. A dollar budget
needs a per-call price (the engine counts calls; a tool-using agent's price
Expand Down
72 changes: 64 additions & 8 deletions tests/test_plan_probe.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,11 @@ def test_an_expired_subscription_fails_the_plan_before_a_run(tmp_path, endpoint)
# One call, and no run store entry: the whole point of the flag.
assert _Handler.hits == 1
assert _store_is_empty(store)
# `next:` stays the last thing a reader sees.
assert out.strip().splitlines()[-1].startswith("next: agentdescent evolve ")
# `next:` stays last, but it is the repair action, not evolve (#1093).
last = out.strip().splitlines()[-1]
assert last.startswith("next: renew the subscription")
assert "next: agentdescent evolve " not in out
assert f"then: agentdescent plan {tmp_path / 'spec.json'} --probe" in out


def test_a_runnable_endpoint_reports_ok_and_exits_zero(tmp_path, endpoint):
Expand Down Expand Up @@ -227,10 +230,10 @@ def _prompt(tmp_path):
return str(path)


def _algorithms_run_args(tmp_path, *, extra=()):
def _algorithms_run_args(tmp_path, *, extra=(), base_url="http://127.0.0.1:1"):
return ["algorithms", "run", "gepa", _prompt(tmp_path), _cases(tmp_path),
"--agent", "openai_compatible", "--model", "m",
"--key-env", "FAKE_KEY", "--base-url", "http://127.0.0.1:1",
"--key-env", "FAKE_KEY", "--base-url", base_url,
"--rounds", "1", "--workers", "1",
"--out", str(tmp_path / "algo.spec.json"), *extra]

Expand All @@ -239,24 +242,32 @@ def test_algorithms_run_json_exits_nonzero_on_a_refused_probe(tmp_path, monkeypa
endpoint):
"""A host gates on this exit code before it starts the run (#1048 review)."""
endpoint(body=ARK_SUBSCRIPTION)
monkeypatch.setenv("FAKE_BASE_URL", _Handler.url)

code, out, err = _run(_algorithms_run_args(tmp_path, extra=["--json", "--probe"]))
code, out, err = _run(_algorithms_run_args(
tmp_path, extra=["--json", "--probe"], base_url=_Handler.url))

payload = json.loads(out)
assert payload["probe"]["ok"] is False
assert payload["ok"] is True, "the plan itself is fine; only the probe refused"
assert payload["ready"] is False
assert payload["next"] == (
"renew the subscription at the provider, or point api_key_env / "
"model at another plan")
assert "evolve" not in payload["next"]
assert code == 1, (code, err)


def test_algorithms_run_human_form_exits_nonzero_too(tmp_path, monkeypatch, endpoint):
endpoint(body=ARK_SUBSCRIPTION)
monkeypatch.setenv("FAKE_BASE_URL", _Handler.url)

code, out, err = _run(_algorithms_run_args(tmp_path, extra=["--probe"]))
code, out, err = _run(_algorithms_run_args(
tmp_path, extra=["--probe"], base_url=_Handler.url))

assert code == 1, (code, out, err)
assert "probe: agent failed" in out + err
last = out.strip().splitlines()[-1]
assert last.startswith("next: renew the subscription")
assert "next: agentdescent evolve" not in out


def test_the_probe_caps_the_reply_budget_and_the_wait(tmp_path, monkeypatch, endpoint):
Expand All @@ -279,3 +290,48 @@ def fake_claude(**kwargs):
assert code == 0, (out, err)
assert seen["max_tokens"] == probe_module.PROBE_MAX_TOKENS
assert seen["timeout"] == probe_module.DEFAULT_PROBE_TIMEOUT

def test_plan_json_marks_a_refused_probe_unready_and_keeps_ok(tmp_path, endpoint):
"""`ok` stays about the plan; `ready` and `next` are how a host gates (#1093)."""
endpoint(body=ARK_SUBSCRIPTION)

code, out, err = _run(["plan", _write_spec(tmp_path), "--json", "--probe"])

payload = json.loads(out)
assert code == 1, (code, err)
assert payload["ok"] is True
assert payload["ready"] is False
assert payload["probe"]["ok"] is False
assert payload["next"] == (
"renew the subscription at the provider, or point api_key_env / "
"model at another plan")


def test_a_successful_probe_still_recommends_evolve(tmp_path, endpoint):
endpoint(code=200)
spec = _write_spec(tmp_path)

code, out, err = _run(["plan", spec, "--probe"])
assert code == 0, (out, err)
assert out.strip().splitlines()[-1].startswith("next: agentdescent evolve ")
assert "then: " not in out

json_code, json_out, json_err = _run(["plan", spec, "--json", "--probe"])
payload = json.loads(json_out)
assert json_code == 0, (json_out, json_err)
assert payload["ok"] is True
assert payload["ready"] is True
assert payload["probe"]["ok"] is True
assert "next" not in payload


def test_without_the_flag_json_has_no_ready_field(tmp_path, endpoint):
endpoint(code=200)

code, out, err = _run(["plan", _write_spec(tmp_path), "--json"])

assert code == 0, (out, err)
payload = json.loads(out)
assert "probe" not in payload
assert "ready" not in payload
assert "next" not in payload
6 changes: 4 additions & 2 deletions tools/gen_algorithms_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,10 @@ def render() -> str:
"reflector before the run it recommends, so a key, endpoint, model or",
"subscription that cannot work is refused there instead of inside",
"`evolve` -- with the same `SubscriptionInactive` / `QuotaExhausted` /",
"`ProviderConfigError` answer and no run store entry. It is off by default:",
"plain `plan` still runs and spends nothing.",
"`ProviderConfigError` answer and no run store entry. A refused probe",
"ends with `next:` naming the repair (not `evolve`), and `--json`",
"sets `ready: false` while `ok` stays about the plan itself (#1093).",
"It is off by default: plain `plan` still runs and spends nothing.",
"",
"Include cases the seed gets wrong: if the seed already scores",
"the target on held-out, the run stops at once with",
Expand Down
Loading