From b7257c959a110205f03c4c18bdcfe8f5b4250fb1 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Mon, 16 Nov 2026 06:44:49 +0800 Subject: [PATCH] fix(cli): a failed plan --probe must not recommend evolve (#1093) When --probe refuses, the last next: line and the JSON next field are the repair action, not evolve. JSON also sets ready: false while ok stays about the plan itself. A successful or omitted probe is unchanged. --- agentdescent/cli.py | 55 ++++++++++++++++---- changelog.d/1093-probe-failed-next.md | 5 ++ docs/algorithms.md | 6 ++- docs/plugin-quickstart.md | 5 +- tests/test_plan_probe.py | 72 ++++++++++++++++++++++++--- tools/gen_algorithms_docs.py | 6 ++- 6 files changed, 125 insertions(+), 24 deletions(-) create mode 100644 changelog.d/1093-probe-failed-next.md diff --git a/agentdescent/cli.py b/agentdescent/cli.py index ffa0a6f3..c4db5c7b 100644 --- a/agentdescent/cli.py +++ b/agentdescent/cli.py @@ -2381,10 +2381,19 @@ def _plan_agent_line(spec: Dict[str, Any]) -> Optional[str]: return line +def _first_failed_probe_next(probe: Mapping[str, Any]) -> str: + """The repair action from the first refused probe row (#1093).""" + for row in probe.get("rows") or []: + if row.get("state") == "failed" and row.get("next"): + return str(row["next"]) + return "run `agentdescent doctor`, then re-run plan --probe" + + def _print_plan(payload: Dict[str, Any], *, spec_path: str, next_line: Optional[str] = None, printed: Optional[set] = None, - probe_lines: Optional[Sequence[str]] = None) -> None: + probe_lines: Optional[Sequence[str]] = None, + then_line: Optional[str] = None) -> None: """Print the human `plan` summary; warnings and the held-out note go to stderr. `payload` is exactly the object `--json` prints, so both surfaces carry the @@ -2429,7 +2438,11 @@ def _print_plan(payload: Dict[str, Any], *, spec_path: str, lines.append(f"note: (+{len(other_notes) - 3} more; --json lists every note)") # The preflight (#1048) sits between the plan and the step it qualifies: # `next:` stays the last line, so a reader who stops early still sees it. + # When the probe refused, that last line is the repair action, not evolve + # (#1093); `then:` names the command that re-runs the probe after the fix. lines.extend(probe_lines or ()) + if then_line: + lines.append(f"then: {then_line}") lines.append(f"next: {next_line or f'agentdescent evolve {spec_path}'}") print("\n".join(lines)) @@ -2497,24 +2510,35 @@ def cmd_plan(a: argparse.Namespace) -> int: print(f"error: {payload.get('error', 'plan failed')}", file=sys.stderr) return 2 probe = _probe_if_asked(a, spec) if payload.get("ok", True) else None + # Default next is evolve; a refused probe replaces it with the repair + # action so a reader (or a host following `next`) does not start a run + # that cannot work (#1093). `ok` stays about the plan/spec itself. + next_line = f"agentdescent evolve {a.spec}" + # Keep an ad-hoc `--algorithm` / `--param` override in the suggested + # command: it lives in memory, not in the spec file. + if getattr(a, "algorithm", None): + next_line += f" --algorithm {a.algorithm}" + for param in getattr(a, "param", None) or []: + next_line += f" --param {param}" + then_line = None if probe is not None: payload["probe"] = probe + payload["ready"] = bool(probe.get("ok", True)) + if not payload["ready"]: + next_line = _first_failed_probe_next(probe) + payload["next"] = next_line + then_line = f"agentdescent plan {a.spec} --probe" if a.json: _out(payload, as_json=True) else: # The human form is the default; `--json` above is the machine form and - # stays exactly the object it always was (#919). - next_line = f"agentdescent evolve {a.spec}" - # Keep an ad-hoc `--algorithm` / `--param` override in the suggested - # command: it lives in memory, not in the spec file. - if getattr(a, "algorithm", None): - next_line += f" --algorithm {a.algorithm}" - for param in getattr(a, "param", None) or []: - next_line += f" --param {param}" + # stays exactly the object it always was (#919) when the probe is not + # asked. `ready` is added only when `--probe` ran. from .probe import probe_lines as _probe_lines _print_plan(payload, spec_path=a.spec, next_line=next_line, - probe_lines=(_probe_lines(probe) if probe is not None else None)) + probe_lines=(_probe_lines(probe) if probe is not None else None), + then_line=then_line) if not payload.get("ok", True): return 2 # A refused probe is a config answer, not a spec error: the same non-zero @@ -4558,11 +4582,19 @@ def _cmd_algorithms_run(a: argparse.Namespace) -> int: report = obj.get("_report") or [] ok = bool(obj.get("ok", True)) probe = None + then_line = None if getattr(a, "probe", False) and stage == "done" and ok: # The same preflight `plan --probe` runs, on the spec this verb just # wrote: one call, before the `evolve` it recommends (#1048). probe = _probe_if_asked(a, load_spec(obj["spec_path"])) obj["probe"] = probe + # `ok` stays about the written plan; `ready` is whether evolve can + # start. A refused probe also replaces `next` so a host does not + # follow it into a run that cannot work (#1093). + obj["ready"] = bool(probe.get("ok", True)) + if not obj["ready"]: + obj["next"] = _first_failed_probe_next(probe) + then_line = f"agentdescent plan {obj['spec_path']} --probe" if as_json: # Exactly one JSON object on stdout for success and error (#674). _out(_public(obj), as_json=True) @@ -4603,7 +4635,8 @@ def _cmd_algorithms_run(a: argparse.Namespace) -> int: from .probe import probe_lines as _probe_lines _print_plan(payload, spec_path=out, next_line=obj["next"], printed=printed, - probe_lines=(_probe_lines(probe) if probe is not None else None)) + probe_lines=(_probe_lines(probe) if probe is not None else None), + then_line=then_line) calls = (payload.get("estimate") or {}).get("agent_calls_upper_bound") if (getattr(a, "rounds", None) is None and getattr(a, "workers", None) is None and isinstance(calls, (int, float)) and calls > QUICK_TRIAL_CALLS): diff --git a/changelog.d/1093-probe-failed-next.md b/changelog.d/1093-probe-failed-next.md new file mode 100644 index 00000000..45b0edde --- /dev/null +++ b/changelog.d/1093-probe-failed-next.md @@ -0,0 +1,5 @@ +Fixed +A failed `plan --probe` or `algorithms run --probe` no longer ends with +`next: agentdescent evolve`. The last line (and the JSON `next`) is the +repair action, `--json` sets `ready: false` while leaving `ok` about the +plan itself, and a successful or omitted probe is unchanged (#1093). diff --git a/docs/algorithms.md b/docs/algorithms.md index c6bbf42f..359f7f7e 100644 --- a/docs/algorithms.md +++ b/docs/algorithms.md @@ -23,8 +23,10 @@ agentdescent init PATH --algorithm NAME --param KEY=VALUE [--rounds N] [--worker reflector before the run it recommends, so a key, endpoint, model or subscription that cannot work is refused there instead of inside `evolve` -- with the same `SubscriptionInactive` / `QuotaExhausted` / -`ProviderConfigError` answer and no run store entry. It is off by default: -plain `plan` still runs and spends nothing. +`ProviderConfigError` answer and no run store entry. A refused probe +ends with `next:` naming the repair (not `evolve`), and `--json` +sets `ready: false` while `ok` stays about the plan itself (#1093). +It is off by default: plain `plan` still runs and spends nothing. Include cases the seed gets wrong: if the seed already scores the target on held-out, the run stops at once with diff --git a/docs/plugin-quickstart.md b/docs/plugin-quickstart.md index ffa1b829..fa2802a1 100644 --- a/docs/plugin-quickstart.md +++ b/docs/plugin-quickstart.md @@ -279,7 +279,10 @@ Two habits worth keeping from the first run: * `--probe` when the run is remote and the answer has to be known now: it sends one minimal call per model-backed agent and reflector, so an expired plan, a wrong model name or a dead endpoint is refused before a run store entry is - written. CLI agents are reported as `probe: … skipped (cli agent)` because one + written. If the probe fails, the last line is the repair action (not + `next: agentdescent evolve`); `then:` is `plan --probe` to re-test, + and `--json` sets `ready: false` while leaving `ok` about the plan itself. + CLI agents are reported as `probe: … skipped (cli agent)` because one call would cost a whole session. * `--budget 5 --usd-per-call 0.02` on the first real run. A dollar budget needs a per-call price (the engine counts calls; a tool-using agent's price diff --git a/tests/test_plan_probe.py b/tests/test_plan_probe.py index a0ba1116..589dcfbe 100644 --- a/tests/test_plan_probe.py +++ b/tests/test_plan_probe.py @@ -127,8 +127,11 @@ def test_an_expired_subscription_fails_the_plan_before_a_run(tmp_path, endpoint) # One call, and no run store entry: the whole point of the flag. assert _Handler.hits == 1 assert _store_is_empty(store) - # `next:` stays the last thing a reader sees. - assert out.strip().splitlines()[-1].startswith("next: agentdescent evolve ") + # `next:` stays last, but it is the repair action, not evolve (#1093). + last = out.strip().splitlines()[-1] + assert last.startswith("next: renew the subscription") + assert "next: agentdescent evolve " not in out + assert f"then: agentdescent plan {tmp_path / 'spec.json'} --probe" in out def test_a_runnable_endpoint_reports_ok_and_exits_zero(tmp_path, endpoint): @@ -227,10 +230,10 @@ def _prompt(tmp_path): return str(path) -def _algorithms_run_args(tmp_path, *, extra=()): +def _algorithms_run_args(tmp_path, *, extra=(), base_url="http://127.0.0.1:1"): return ["algorithms", "run", "gepa", _prompt(tmp_path), _cases(tmp_path), "--agent", "openai_compatible", "--model", "m", - "--key-env", "FAKE_KEY", "--base-url", "http://127.0.0.1:1", + "--key-env", "FAKE_KEY", "--base-url", base_url, "--rounds", "1", "--workers", "1", "--out", str(tmp_path / "algo.spec.json"), *extra] @@ -239,24 +242,32 @@ def test_algorithms_run_json_exits_nonzero_on_a_refused_probe(tmp_path, monkeypa endpoint): """A host gates on this exit code before it starts the run (#1048 review).""" endpoint(body=ARK_SUBSCRIPTION) - monkeypatch.setenv("FAKE_BASE_URL", _Handler.url) - code, out, err = _run(_algorithms_run_args(tmp_path, extra=["--json", "--probe"])) + code, out, err = _run(_algorithms_run_args( + tmp_path, extra=["--json", "--probe"], base_url=_Handler.url)) payload = json.loads(out) assert payload["probe"]["ok"] is False assert payload["ok"] is True, "the plan itself is fine; only the probe refused" + assert payload["ready"] is False + assert payload["next"] == ( + "renew the subscription at the provider, or point api_key_env / " + "model at another plan") + assert "evolve" not in payload["next"] assert code == 1, (code, err) def test_algorithms_run_human_form_exits_nonzero_too(tmp_path, monkeypatch, endpoint): endpoint(body=ARK_SUBSCRIPTION) - monkeypatch.setenv("FAKE_BASE_URL", _Handler.url) - code, out, err = _run(_algorithms_run_args(tmp_path, extra=["--probe"])) + code, out, err = _run(_algorithms_run_args( + tmp_path, extra=["--probe"], base_url=_Handler.url)) assert code == 1, (code, out, err) assert "probe: agent failed" in out + err + last = out.strip().splitlines()[-1] + assert last.startswith("next: renew the subscription") + assert "next: agentdescent evolve" not in out def test_the_probe_caps_the_reply_budget_and_the_wait(tmp_path, monkeypatch, endpoint): @@ -279,3 +290,48 @@ def fake_claude(**kwargs): assert code == 0, (out, err) assert seen["max_tokens"] == probe_module.PROBE_MAX_TOKENS assert seen["timeout"] == probe_module.DEFAULT_PROBE_TIMEOUT + +def test_plan_json_marks_a_refused_probe_unready_and_keeps_ok(tmp_path, endpoint): + """`ok` stays about the plan; `ready` and `next` are how a host gates (#1093).""" + endpoint(body=ARK_SUBSCRIPTION) + + code, out, err = _run(["plan", _write_spec(tmp_path), "--json", "--probe"]) + + payload = json.loads(out) + assert code == 1, (code, err) + assert payload["ok"] is True + assert payload["ready"] is False + assert payload["probe"]["ok"] is False + assert payload["next"] == ( + "renew the subscription at the provider, or point api_key_env / " + "model at another plan") + + +def test_a_successful_probe_still_recommends_evolve(tmp_path, endpoint): + endpoint(code=200) + spec = _write_spec(tmp_path) + + code, out, err = _run(["plan", spec, "--probe"]) + assert code == 0, (out, err) + assert out.strip().splitlines()[-1].startswith("next: agentdescent evolve ") + assert "then: " not in out + + json_code, json_out, json_err = _run(["plan", spec, "--json", "--probe"]) + payload = json.loads(json_out) + assert json_code == 0, (json_out, json_err) + assert payload["ok"] is True + assert payload["ready"] is True + assert payload["probe"]["ok"] is True + assert "next" not in payload + + +def test_without_the_flag_json_has_no_ready_field(tmp_path, endpoint): + endpoint(code=200) + + code, out, err = _run(["plan", _write_spec(tmp_path), "--json"]) + + assert code == 0, (out, err) + payload = json.loads(out) + assert "probe" not in payload + assert "ready" not in payload + assert "next" not in payload diff --git a/tools/gen_algorithms_docs.py b/tools/gen_algorithms_docs.py index de06f8fd..9df89d5e 100644 --- a/tools/gen_algorithms_docs.py +++ b/tools/gen_algorithms_docs.py @@ -90,8 +90,10 @@ def render() -> str: "reflector before the run it recommends, so a key, endpoint, model or", "subscription that cannot work is refused there instead of inside", "`evolve` -- with the same `SubscriptionInactive` / `QuotaExhausted` /", - "`ProviderConfigError` answer and no run store entry. It is off by default:", - "plain `plan` still runs and spends nothing.", + "`ProviderConfigError` answer and no run store entry. A refused probe", + "ends with `next:` naming the repair (not `evolve`), and `--json`", + "sets `ready: false` while `ok` stays about the plan itself (#1093).", + "It is off by default: plain `plan` still runs and spends nothing.", "", "Include cases the seed gets wrong: if the seed already scores", "the target on held-out, the run stops at once with",