Capture and invigilate harmonization eval traces

.github/workflows/run-eval.yml

@@ -123,6 +123,19 @@ jobs:
           uv run python -m src.folds.stage_raw_data --indices "$STAGE_INDICES" \
             --dest "$RUN_ENV/inputs/raw" --drive-method public
 
+      - name: Preserve the invigilator outside the agent workspace
+        # The following step deliberately deletes the checkout, including its Python
+        # environment. Preserve only the auditor and its already-synced runtime
+        # outside the agent's filesystem boundary for the post-run audit.
+        env:
+          AUDITOR_DIR: ${{ runner.temp }}/harmonization-invigilator
+        run: |
+          set -euo pipefail
+          test -x .venv/bin/python
+          mkdir -p "$AUDITOR_DIR"
+          cp src/folds/invigilator.py "$AUDITOR_DIR/invigilator.py"
+          mv .venv "$AUDITOR_DIR/venv"
+
       - name: Replace checkout with the isolated agent workspace
         # This is the isolation boundary.  Build/staging happen in the trusted
         # checkout; immediately before Claude starts, replace that checkout
@@ -134,6 +147,7 @@ jobs:
           rsync -a --delete "$RUN_ENV/" "$GITHUB_WORKSPACE/"
 
       - name: Run harmonization agent in the run environment
+        id: harmonize
         # Cap runaway evals so a hung/looping agent can't burn the full 6h limit.
         timeout-minutes: 60
         uses: anthropics/claude-code-action@beta
@@ -161,6 +175,29 @@ jobs:
             directories, absolute paths outside it, network services, APIs, or other
             external data sources. Write every deliverable under `output/`.
 
+      - name: Capture and invigilate the agent trace
+        # The action's execution_file contains its complete tool-use log. Keep
+        # the original JSON in the isolated workspace and fail the run on any
+        # access outside that workspace. This step is intentionally before
+        # branch staging, so the audit result is included in the staged run.
+        if: always()
+        env:
+          AUDITOR_DIR: ${{ runner.temp }}/harmonization-invigilator
+          EXECUTION_FILE: ${{ steps.harmonize.outputs.execution_file }}
+        run: |
+          set -euo pipefail
+          mkdir -p audit
+          if [ -z "$EXECUTION_FILE" ] || [ ! -f "$EXECUTION_FILE" ]; then
+            echo "ERROR: Claude action did not provide an execution trace." \
+              | tee audit/invigilator_report.txt
+            exit 1
+          fi
+          cp "$EXECUTION_FILE" audit/claude-execution.json
+          "$AUDITOR_DIR/venv/bin/python" "$AUDITOR_DIR/invigilator.py" \
+            --trace audit/claude-execution.json --env "$GITHUB_WORKSPACE" \
+            --repo-root "$GITHUB_WORKSPACE" \
+            | tee audit/invigilator_report.txt
+
       - name: Stage completed isolated environment on an eval branch
         # This deliberately runs only after the agent. At this point the current
         # workspace is the answer-free fold (not the trusted checkout), so an

README.md

@@ -185,7 +185,7 @@ substituting the fold name. The agent writes its deliverables to:
 /tmp/data-harmonization-eval-runs/fold-02-holdout-15-26/output/
 ```
 
-If your agent provider records tool calls as JSONL, audit the run afterward:
+For a locally run agent, audit its tool-use JSONL afterward:
 
 ```bash
 uv run python -m src.folds.invigilator \
@@ -204,7 +204,12 @@ finishes, it creates a new orphan branch named
 `eval/<fold-name>-<run-id>-<attempt>` containing that isolated workspace, then
 uploads the same workspace as an `eval-<fold-id>` artifact. The branch is
 created only after the agent run and has no parent commit, so it does not carry
-the trusted checkout history or its held-out gold artifacts.
+the trusted checkout history or its held-out gold artifacts. The workflow also
+copies Claude Code's execution log to `audit/claude-execution.json`, runs the
+invigilator before staging, and records its report in
+`audit/invigilator_report.txt`. A missing trace or any out-of-bound access fails
+the workflow; the resulting audit artifacts are still retained in the eval
+branch and uploaded artifact for review.
 
 ### Results and scoring
 

src/folds/invigilator.py

@@ -65,7 +65,13 @@
 
 
 def load_tool_uses(trace_path: Path) -> list[tuple[str, dict]]:
-    """Extract ``(tool_name, tool_input)`` pairs from a JSONL agent trace."""
+    """Extract ``(tool_name, tool_input)`` pairs from a JSON or JSONL trace.
+
+    Claude Code session transcripts are JSONL, while the GitHub action's
+    ``execution_file`` output is a single JSON document. Supporting both lets
+    the evaluator archive the action output verbatim rather than reserializing
+    a partial view of it before audit.
+    """
     out: list[tuple[str, dict]] = []
 
     def walk(o):
@@ -78,10 +84,15 @@ def walk(o):
             for v in o:
                 walk(v)
 
-    for line in Path(trace_path).read_text().splitlines():
-        line = line.strip()
-        if line:
-            walk(json.loads(line))
+    text = Path(trace_path).read_text().strip()
+    if not text:
+        raise ValueError(f"trace is empty: {trace_path}")
+    try:
+        records = [json.loads(text)]
+    except json.JSONDecodeError:
+        records = [json.loads(line) for line in text.splitlines() if line.strip()]
+    for record in records:
+        walk(record)
     return out
 
 

tests/test_invigilator.py

@@ -6,7 +6,7 @@
 
 from typer.testing import CliRunner
 
-from src.folds.invigilator import app, audit, lexical_resolve, under
+from src.folds.invigilator import app, audit, lexical_resolve, load_tool_uses, under
 
 
 def write_trace(path: Path, tool_uses: list[tuple[str, dict]]) -> Path:
@@ -20,6 +20,15 @@ def write_trace(path: Path, tool_uses: list[tuple[str, dict]]) -> Path:
     return path
 
 
+def test_load_tool_uses_accepts_action_execution_json(tmp_path):
+    """The GitHub action emits one JSON document rather than JSONL."""
+    trace = tmp_path / "execution.json"
+    trace.write_text(json.dumps({
+        "messages": [{"type": "tool_use", "name": "Read", "input": {"file_path": "x"}}]
+    }))
+    assert load_tool_uses(trace) == [("Read", {"file_path": "x"})]
+
+
 def make_repo(tmp_path):
     repo = tmp_path
     env = repo / ".runs" / "cfg"