diff --git a/.github/scripts/check_plugin_api.py b/.github/scripts/check_plugin_api.py index b0ee5462..b19726fc 100644 --- a/.github/scripts/check_plugin_api.py +++ b/.github/scripts/check_plugin_api.py @@ -32,6 +32,7 @@ "agent_env.providers.env_state.env_state_provider.EnvStateProvider", "agent_env.providers.env_providers.env_provider.EnvironmentProvider", "agent_env.explorer.plugin.ExplorerPlugin", + "agent_env.runner.runner.Runner", ) USED = ( "agent_env.task_step.context.TaskStepContext", "agent_env.plugins", @@ -39,6 +40,11 @@ # What a plugin reads of recorded runs. "agent_env.task.store.task_instances", "agent_env.task.store.count_task_instances", "agent_env.task.store.find_task_instance", "agent_env.task.store.TaskInstance", + # What a runner returns and the run index it records each run in. + "agent_env.runner.runner.RunStatus", "agent_env.runner.runner.RunHandle", "agent_env.runner.runner.RunRecord", + "agent_env.runner.store.ensure_indexes", "agent_env.runner.store.insert_run", "agent_env.runner.store.get_run", + "agent_env.runner.store.list_runs", "agent_env.runner.store.count_runs", "agent_env.runner.store.active_runs", + "agent_env.runner.store.set_running", "agent_env.runner.store.mark_terminal", # What an environment provider reads to deploy a built-in env. "agent_env.env.envs.mcp_server.MCPServerEnv.docker_image_artifact", "agent_env.env.envs.mcp_server.MCPServerEnv.environment_name", "agent_env.env.envs.website.WebsiteEnv.backend_docker_image_artifact", diff --git a/AGENTS.md b/AGENTS.md index 57f012f5..e62bdcf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,7 +104,7 @@ and deserialized through a registry. | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | | `a2a_agent/` | The `A2AAgent` entity (`a2a_agent`), its stores and the validator steps. The protocol package provides the agent-side framework. | -| `runner/` | The `[runner]` seam: `Runner.submit()` returns a `RunHandle` (`run_id`, `instance_id`) before the run starts; `LocalRunner` is built in. | +| `runner/` | The `[runner]` seam: `Runner.submit()` returns a `RunHandle` (`run_id`, `instance_id`) before the run starts; `LocalRunner` is built in. `runner/store.py` is the run index every runner writes and the explorer reads. | | `bundle/` | A bundle is a folder of entities (`envs/`, `agents/`, `artifacts/`, `skills/`, `tasks/.json`, `evals/.toml`) that runs without a registry: `agent-env run` plans it, writes it into the `@local` namespace (reusing each version whose inputs haven't changed) and runs its evals or tasks; `agent-env bundle init` writes a stored task or eval as one, with its references pinned. | | `eval/` | `Eval` (`eval`), a versioned list of task references, and its store. | | `plugins/` | Entry-point plugins (see Extension points): the group names, `inventory()`, which `agent-env plugin list/show/check` report, `load_failures` and `settings`. The top level is public; the underscored modules are internal. | @@ -164,6 +164,10 @@ orchestrators call it the same way. What callers rely on: through `record_task_failure()`, retries through `undo_steps()`. - The runner writes its run identifier to `context.metadata["workflow_id"]` before calling `run()`; core stores it opaquely with the instance. +- A runner records every run it accepts in the run index (`agent_env.runner.store`): `insert_run` at + submit, `set_running` when it starts, `mark_terminal` when it ends. The explorer lists, counts and + groups runs from that collection and never calls the runner to ask, so a run missing there is + invisible and one never marked terminal shows as in flight. - Per-run overrides travel in `context.metadata["user_overrides"]` (`litellm_api_key`, `a2a_agent_id`, `agent_sandbox`, `env_sandbox`, `env_state_type`, and more), never in `os.environ`; steps fall back to the configured `[model]` key. `step_params` maps a step id to overrides of that diff --git a/src/agent_env/explorer/routers/runs.py b/src/agent_env/explorer/routers/runs.py index 3241e18e..f8da1d0f 100644 --- a/src/agent_env/explorer/routers/runs.py +++ b/src/agent_env/explorer/routers/runs.py @@ -235,7 +235,7 @@ def list_runs( from agent_env.runner import store as run_store records = run_store.list_runs(task_id=task_id, limit=limit, offset=offset) - total = docs().count("runs", Filter.of(task_id=task_id)) + total = run_store.count_runs(task_id) return PaginatedResponse( items=[r.to_dict() for r in records], total=total, limit=limit, offset=offset, has_more=offset + len(records) < total, diff --git a/src/agent_env/runner/runner.py b/src/agent_env/runner/runner.py index 321df72e..1771e68f 100644 --- a/src/agent_env/runner/runner.py +++ b/src/agent_env/runner/runner.py @@ -87,7 +87,7 @@ def from_dict(cls, data: dict) -> "RunRecord": class Runner(ABC): - """Dispatches task runs and reports on them. + """Dispatches task runs and reports on them, recording each run in ``agent_env.runner.store``. ``type`` is persisted on every run record: a hub restarted under a different ``[runner]`` reports on a run it does not own rather than misreporting it. diff --git a/src/agent_env/runner/store.py b/src/agent_env/runner/store.py index 884bfb28..d30b0b6c 100644 --- a/src/agent_env/runner/store.py +++ b/src/agent_env/runner/store.py @@ -1,9 +1,8 @@ -"""Run-record persistence for the in-process runner. +"""The run index every ``[runner]`` writes: ``insert_run`` at submit, ``set_running`` when the run +starts, ``mark_terminal`` when it ends. The explorer lists, counts and groups runs from here and never +asks the runner, so a run missing here is invisible and one never marked terminal stays in flight. -Stored as a collection so the hub can list runs and report their status — not a durable -work queue: the runner executes ``Task.run()`` in-process and holds the run's asyncio task -in memory, so there is no lease or claim here. A run left non-terminal by a dead process -is failed at ``start()``, not resumed. A durable, lease-based runner is a follow-up. +It is an index, not a work queue: there is no lease or claim here. """ from __future__ import annotations @@ -59,6 +58,10 @@ def list_runs( return [RunRecord.from_dict(d) for d in docs] +def count_runs(task_id: Optional[str] = None) -> int: + return _docs().count(RUNS_COLLECTION, Filter.of(task_id=task_id) if task_id else Filter()) + + def active_runs(runner: str, limit: int = 100_000) -> list[RunRecord]: """Every non-terminal (QUEUED/RUNNING) run for ``runner`` — used at startup to reconcile orphans a dead process left behind, no matter how many terminal runs precede them.""" diff --git a/tst/unit/runner/run_store_test.py b/tst/unit/runner/run_store_test.py index f167fe6d..5c1eff2a 100644 --- a/tst/unit/runner/run_store_test.py +++ b/tst/unit/runner/run_store_test.py @@ -1,4 +1,4 @@ -"""Run-store invariants for the in-process runner: a terminal run is never overwritten, +"""Run-index invariants: a terminal run is never overwritten, and set_running only advances a still-queued one.""" import pytest @@ -48,3 +48,11 @@ def test_set_running_advances_only_a_queued_run(docs): run_store.insert_run(_record("c", RunStatus.CANCELED)) run_store.set_running("c") assert run_store.get_run("c").status == RunStatus.CANCELED + + +def test_count_runs_counts_one_task_or_every_task(docs): + run_store.insert_run(_record("a", RunStatus.QUEUED)) + run_store.insert_run(RunRecord(run_id="b", runner="other", task_id="u", task_version=1, instance_id="j")) + assert run_store.count_runs("t") == 1 + assert run_store.count_runs("missing") == 0 + assert run_store.count_runs() == 2