diff --git a/.github/workflows/run-eval.yml b/.github/workflows/run-eval.yml index e87fe09..4c60126 100644 --- a/.github/workflows/run-eval.yml +++ b/.github/workflows/run-eval.yml @@ -123,6 +123,19 @@ jobs: uv run python -m src.folds.stage_raw_data --indices "$STAGE_INDICES" \ --dest "$RUN_ENV/inputs/raw" --drive-method public + - name: Preserve the invigilator outside the agent workspace + # The following step deliberately deletes the checkout, including its Python + # environment. Preserve only the auditor and its already-synced runtime + # outside the agent's filesystem boundary for the post-run audit. + env: + AUDITOR_DIR: ${{ runner.temp }}/harmonization-invigilator + run: | + set -euo pipefail + test -x .venv/bin/python + mkdir -p "$AUDITOR_DIR" + cp src/folds/invigilator.py "$AUDITOR_DIR/invigilator.py" + mv .venv "$AUDITOR_DIR/venv" + - name: Replace checkout with the isolated agent workspace # This is the isolation boundary. Build/staging happen in the trusted # checkout; immediately before Claude starts, replace that checkout @@ -134,6 +147,7 @@ jobs: rsync -a --delete "$RUN_ENV/" "$GITHUB_WORKSPACE/" - name: Run harmonization agent in the run environment + id: harmonize # Cap runaway evals so a hung/looping agent can't burn the full 6h limit. timeout-minutes: 60 uses: anthropics/claude-code-action@beta @@ -161,6 +175,29 @@ jobs: directories, absolute paths outside it, network services, APIs, or other external data sources. Write every deliverable under `output/`. + - name: Capture and invigilate the agent trace + # The action's execution_file contains its complete tool-use log. Keep + # the original JSON in the isolated workspace and fail the run on any + # access outside that workspace. This step is intentionally before + # branch staging, so the audit result is included in the staged run. + if: always() + env: + AUDITOR_DIR: ${{ runner.temp }}/harmonization-invigilator + EXECUTION_FILE: ${{ steps.harmonize.outputs.execution_file }} + run: | + set -euo pipefail + mkdir -p audit + if [ -z "$EXECUTION_FILE" ] || [ ! -f "$EXECUTION_FILE" ]; then + echo "ERROR: Claude action did not provide an execution trace." \ + | tee audit/invigilator_report.txt + exit 1 + fi + cp "$EXECUTION_FILE" audit/claude-execution.json + "$AUDITOR_DIR/venv/bin/python" "$AUDITOR_DIR/invigilator.py" \ + --trace audit/claude-execution.json --env "$GITHUB_WORKSPACE" \ + --repo-root "$GITHUB_WORKSPACE" \ + | tee audit/invigilator_report.txt + - name: Stage completed isolated environment on an eval branch # This deliberately runs only after the agent. At this point the current # workspace is the answer-free fold (not the trusted checkout), so an diff --git a/README.md b/README.md index 16c520d..93a58a0 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ substituting the fold name. The agent writes its deliverables to: /tmp/data-harmonization-eval-runs/fold-02-holdout-15-26/output/ ``` -If your agent provider records tool calls as JSONL, audit the run afterward: +For a locally run agent, audit its tool-use JSONL afterward: ```bash uv run python -m src.folds.invigilator \ @@ -204,7 +204,12 @@ finishes, it creates a new orphan branch named `eval/--` containing that isolated workspace, then uploads the same workspace as an `eval-` artifact. The branch is created only after the agent run and has no parent commit, so it does not carry -the trusted checkout history or its held-out gold artifacts. +the trusted checkout history or its held-out gold artifacts. The workflow also +copies Claude Code's execution log to `audit/claude-execution.json`, runs the +invigilator before staging, and records its report in +`audit/invigilator_report.txt`. A missing trace or any out-of-bound access fails +the workflow; the resulting audit artifacts are still retained in the eval +branch and uploaded artifact for review. ### Results and scoring diff --git a/src/folds/invigilator.py b/src/folds/invigilator.py index 58c1705..0dc5244 100644 --- a/src/folds/invigilator.py +++ b/src/folds/invigilator.py @@ -65,7 +65,13 @@ def load_tool_uses(trace_path: Path) -> list[tuple[str, dict]]: - """Extract ``(tool_name, tool_input)`` pairs from a JSONL agent trace.""" + """Extract ``(tool_name, tool_input)`` pairs from a JSON or JSONL trace. + + Claude Code session transcripts are JSONL, while the GitHub action's + ``execution_file`` output is a single JSON document. Supporting both lets + the evaluator archive the action output verbatim rather than reserializing + a partial view of it before audit. + """ out: list[tuple[str, dict]] = [] def walk(o): @@ -78,10 +84,15 @@ def walk(o): for v in o: walk(v) - for line in Path(trace_path).read_text().splitlines(): - line = line.strip() - if line: - walk(json.loads(line)) + text = Path(trace_path).read_text().strip() + if not text: + raise ValueError(f"trace is empty: {trace_path}") + try: + records = [json.loads(text)] + except json.JSONDecodeError: + records = [json.loads(line) for line in text.splitlines() if line.strip()] + for record in records: + walk(record) return out diff --git a/tests/test_invigilator.py b/tests/test_invigilator.py index afe688a..d8d8d49 100644 --- a/tests/test_invigilator.py +++ b/tests/test_invigilator.py @@ -6,7 +6,7 @@ from typer.testing import CliRunner -from src.folds.invigilator import app, audit, lexical_resolve, under +from src.folds.invigilator import app, audit, lexical_resolve, load_tool_uses, under def write_trace(path: Path, tool_uses: list[tuple[str, dict]]) -> Path: @@ -20,6 +20,15 @@ def write_trace(path: Path, tool_uses: list[tuple[str, dict]]) -> Path: return path +def test_load_tool_uses_accepts_action_execution_json(tmp_path): + """The GitHub action emits one JSON document rather than JSONL.""" + trace = tmp_path / "execution.json" + trace.write_text(json.dumps({ + "messages": [{"type": "tool_use", "name": "Read", "input": {"file_path": "x"}}] + })) + assert load_tool_uses(trace) == [("Read", {"file_path": "x"})] + + def make_repo(tmp_path): repo = tmp_path env = repo / ".runs" / "cfg"