From 422bd7047c11b9c844fdc7c7cf6c62ab1460ce89 Mon Sep 17 00:00:00 2001 From: Sebastian Gonzalez Date: Wed, 11 Nov 2026 11:46:43 -0800 Subject: [PATCH] test: bring public's suites up to the v0.151.0 plugin A promotion replaces plugins/memhub/ and install.sh only, so public's tests/ stayed at 0.133.0 while the plugin moved to 0.151.0. Seventeen suites failed guard and probe-tests on #296, each on an intentional internal change. The seventeen suites, tests/agent_fixture.py, the cursor/rewind fixtures, the readers-cli codex golden, scripts/gen_codex_hooks.py and docs/specs/pr-linking-plugin-spec.md are internal's at e4fcf6d2 (the promoted commit), with memhub-staging rewritten to memhub. codex_correctness_test keeps its staging fixture name so it still proves the prod install wins. The root README gains /memhub:rewind and counts seventeen skills. No file under plugins/ changes. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- docs/specs/pr-linking-plugin-spec.md | 41 +- scripts/gen_codex_hooks.py | 6 +- tests/agent_fixture.py | 318 +++++++++ tests/brain_brief_test.py | 15 +- tests/claude_hook_guard_test.py | 10 +- tests/codex_correctness_test.py | 28 +- tests/codex_hooks_parity_test.py | 54 +- tests/codex_hooks_setup_test.py | 143 +++- tests/cursor_capture_test.py | 39 +- tests/cursor_timestamp_test.py | 265 +++++++- tests/cursor_usage_test.py | 186 +++++- tests/fixtures/cursor_ide_subagent_taps.jsonl | 15 + .../fixtures/cursor_ide_usage_recording.json | 611 ++++++++++++++++++ tests/fixtures/rewind_native_probe.py | 334 ++++++++++ tests/flush_session_test.py | 214 +++++- tests/goldens/readers-cli-codex.jsonl | 10 +- tests/readers_test.py | 262 +++++++- tests/redact_test.py | 79 ++- tests/rulebook_container_test.py | 5 +- tests/rulebook_disclosure_test.py | 61 +- tests/rulebook_hook_test.py | 164 ++++- tests/spec_owns_port_test.py | 8 +- 23 files changed, 2753 insertions(+), 123 deletions(-) create mode 100644 tests/agent_fixture.py create mode 100644 tests/fixtures/cursor_ide_subagent_taps.jsonl create mode 100644 tests/fixtures/cursor_ide_usage_recording.json create mode 100644 tests/fixtures/rewind_native_probe.py diff --git a/README.md b/README.md index d4fcf849..992f927e 100644 --- a/README.md +++ b/README.md @@ -387,7 +387,7 @@ With the variable unset, the default, none of this runs. ## Skills -Sixteen skills ship in `plugins/memhub/skills/` (the deprecated `commands/` +Seventeen skills ship in `plugins/memhub/skills/` (the deprecated `commands/` format is gone; invocation is unchanged). Each is both user-invocable as `/memhub:` and **model-invocable**: saying "save this spec to memhub" or "what did we decide about X?" in plain language triggers the right skill. @@ -420,6 +420,12 @@ format is gone; invocation is unchanged). Each is both user-invocable as sessions that predate capture, or ran while it was dormant. It imports under the session's own id, the same id capture uses, so a session is one conversation rather than two competing copies. +- `/memhub:rewind` — selects a locally checkpointed Codex or Claude Code + message and launches a replacement prompt or model in an isolated checkout, + directly from the client. Studio is optional. Only the repository is + restored; retained extra writable directories and remote effects are not + rewound. Requires prior checkpoint capture and a supported native version + (Claude Code 2.1.322+, with its sandbox available). - `/memhub:save-artifact [name]` — terminal upload of a file as an artifact. Both upload skills exist so the model never re-emits file or transcript content token by token — a helper script ships the bytes. diff --git a/docs/specs/pr-linking-plugin-spec.md b/docs/specs/pr-linking-plugin-spec.md index f91f9491..65939b61 100644 --- a/docs/specs/pr-linking-plugin-spec.md +++ b/docs/specs/pr-linking-plugin-spec.md @@ -11,6 +11,11 @@ is the sole source of truth for the plugin half: an implementer should need noth work type to every link. §4.5's context strings and §8/§9's skill steps below are the PRE-classification shapes; where the two disagree, the work-type spec is authoritative. +**Merged (2026-10):** the two skills below are now one. `/memhub:find-contributing-sessions` is +`/memhub:link-pr --find [pr-or-url ...]`, with unchanged behavior, and its scanner moved to +`skills/link-pr/scripts/find_sessions.py`; the hook text offers `/memhub:link-pr --find `. +The §5 file table and §9.1 keep the old name as the history of what this spec shipped. + --- ## 0. Overview @@ -32,16 +37,16 @@ and `pr_babysit_trigger.py` arms the babysit loop — but neither creates a link 2. *connected, and this call OPENED the pull request* (`gh pr create`, a POST to `…/repos///pulls`, or a create-shaped GitHub MCP tool) → link this session, unconditionally, `link_source="session_self"` — and, only if the code came mostly from - elsewhere, also offer `/memhub:find-contributing-sessions`; + elsewhere, also offer `/memhub:link-pr --find`; 3. *connected, any other GitHub call that names one PR* → the model decides: link if it wrote - the code here, else offer `/memhub:find-contributing-sessions`. + the code here, else offer `/memhub:link-pr --find`. **Opening a pull request is itself work the session did**, so case 2 asks no authorship question — the creating session is always part of that PR's history, and a PR having many sessions means linking it displaces nobody. Case 3 is where judgment lives, and it is the model's, not the hook's: the agent knows whether it edited these files this session, and no heuristic has to. - **`/memhub:link-pr`** — the manual path. Link this session (or named ones) to a PR. -- **`/memhub:find-contributing-sessions`** — scans local session history for sessions that +- **`/memhub:link-pr --find`** (originally its own skill, `find-contributing-sessions`) — scans local session history for sessions that plausibly wrote the code in a PR, ranks them, has the user approve, and links the approved ones. **Unchanged, deliberately:** `pr_provenance.py` and the `provenance.github_pr_urls` capture path @@ -64,7 +69,7 @@ are independent. | `plugins/memhub/scripts/codex_hook_bridge.py` `_dispatch_post` | How Codex fans several hook scripts out and folds their `additionalContext` into one document. | | `plugins/memhub/scripts/cursor_capture.py` | Cursor's launcher. Today it only ever answers `{"permission": "allow"}` — see §5.3. | | `plugins/memhub/scripts/capture.py` + `scripts/readers/` | `list`/`import` over all three hosts. `current` is added here (§7). | -| `plugins/memhub/skills/pr-babysit/SKILL.md` | House style for a skill that takes a PR argument, resolves the repo's room, and is auto-armed by a hook. | +| `plugins/memhub/skills/pr-babysit/SKILL.md` | House style for a skill that takes a PR argument and is auto-armed by a hook. | | `plugins/memhub/skills/start-rulebook/` | The precedent for a skill with its own `scripts/` that scans local transcripts and hands back JSON instead of transcript text. | --- @@ -274,7 +279,7 @@ vary): - `touches_github` when the name splits as `mcp____` and the **server** segment contains "github". The server segment runs to the FIRST `__`, and is not `[^_]*`: server names routinely contain underscores (`github_enterprise`, and every plugin-provided server — this - repo's own tools arrive as `mcp__plugin_memhub_memhub__add_memory`), and a + repo's own tools arrive as `mcp__plugin_memhub_memhub__save_artifact`), and a no-underscore pattern rejected all of them. Matching `github` anywhere in the WHOLE name would instead catch `mcp__notes__github_summary`, which is a note-taking tool — the server segment is what separates the two. @@ -473,7 +478,7 @@ it links itself unconditionally: > the pull request is itself work this session did, and the call is idempotent. > Then, **only if most of the code in this pull request was written in OTHER sessions** (you > opened it for work done earlier, or you are shipping someone else's branch), add one sentence -> offering `/memhub:find-contributing-sessions ` so those sessions get linked too. A pull +> offering `/memhub:link-pr --find ` so those sessions get linked too. A pull > request has many sessions; yours does not displace them. Do not run it without a yes. **B2 — connected, any other GitHub call naming one PR** (`github_connected:true`, @@ -490,7 +495,7 @@ fetching a pull request says nothing about who wrote it: > idempotent. > **If it was not** — you are reviewing, checking out, or commenting on someone else's work, or > work from an earlier session — do not link. Instead offer, in one sentence, to run -> `/memhub:find-contributing-sessions ` to find the sessions that did write it. Do not +> `/memhub:link-pr --find ` to find the sessions that did write it. Do not > run it without the user saying yes. > If this session already linked itself to this pull request, say nothing at all. @@ -514,7 +519,7 @@ cannot silence the other: ```json { - "matcher": "^(Bash|mcp__.*[Gg]it[Hh]ub.*__.*)$", + "matcher": "^(Bash|mcp__.*[Gg]it[Hh]ub.*__.*|mcp__.+__link_pr)$", "hooks": [{ "type": "command", "timeout": 15, @@ -531,10 +536,14 @@ for a check that is almost always going to decline — but it must NOT use `[^_] server segment, which rejects `github_enterprise` and every plugin-provided server. This matcher and the payload globs below are coarse pre-filters that only decide whether the trigger starts; `pr_link.is_github_mcp_tool` is the precise gate, and it is the one that has to be exactly right. +The `mcp__.+__link_pr` alternative is MemHub's own `link_pr`: a successful call with +`pr_type="fix"` earns the bug-origin instruction (MemHub-Backend +`docs/specs/pr-weight-success-spec.md` §7.1); `pr_link.is_memhub_link_pr` is its precise gate, +and `__unlink_pr` matches neither. `scripts/hook_entry.py` runs every Claude hook (the plugin directory admits no inline shell in a hook command), and its `PostToolUse pr_link_trigger` route matches the shell-`case` globs -`*gh*pr*|*[Gg]it[Hh]ub*|*api/v3*|*repos/*pulls*` before anything else runs — against the two +`*gh*pr*|*[Gg]it[Hh]ub*|*api/v3*|*repos/*pulls*|mcp__*__link_pr` before anything else runs — against the two fields `touches_github` reads, the tool name and `tool_input.command` (each also case-folded and with shell quotes and backslashes removed, as `pr_link` reads them), never the tool's output. That cheap byte test keeps `pr_link_trigger.py` from starting on ordinary shell traffic — the same @@ -545,7 +554,8 @@ call; and `*api/v3*` plus `*repos/*pulls*` catch the ENTERPRISE forms, which con `curl -X POST https://ghe.corp/api/v3/repos/o/r/pulls` has no "github" anywhere in it, and `gh api --hostname ghe.corp … repos/o/r/pulls` has no URL at all. Without those two the whole enterprise path is detected by the Python and then never reached, which is a silence no log -explains. A `gh api repos/o/r/pulls` command matches through `*repos/*pulls*` in the command +explains. `mcp__*__link_pr` admits the bug-origin lane, whose tool name carries none of the +other four. A `gh api repos/o/r/pulls` command matches through `*repos/*pulls*` in the command itself. The globs are pinned as a superset of `touches_github` by `tests/hook_entry_test.py`, so tool output — a `cat` of a changelog that cites a PR — no longer starts the trigger at all. @@ -572,7 +582,8 @@ different things: - **`plugins/memhub/hooks/codex-hooks.json`** — the plugin-bundled manifest, read by Codex releases that load hooks from the plugin itself. It is re-fetched with every plugin upgrade and costs the user nothing, so its `PostToolUse` matcher IS widened here, to - `^(Edit|MultiEdit|Write|NotebookEdit|apply_patch|Bash|shell|local_shell|mcp__.*[Gg]it[Hh]ub.*__.*)$`. + `^(Edit|MultiEdit|Write|NotebookEdit|apply_patch|Bash|shell|local_shell|mcp__.*[Gg]it[Hh]ub.*__.*)$` + (since ENG-1268 also `|mcp__.+__link_pr`, routed by `_dispatch_post` to the same trigger). On that path Codex gets GitHub-MCP detection. - **`references/codex-hooks-bridge.json`** — the compatibility bridge `setup_codex_hooks.py` installs into `~/.codex/hooks.json`, a file the user has already trusted. Widening it would @@ -753,7 +764,11 @@ they carry the org resolution. --- -## 9. `/memhub:find-contributing-sessions` +## 9. `/memhub:link-pr --find` (originally `/memhub:find-contributing-sessions`) + +> **Merged (2026-10)** into `/memhub:link-pr` as its `--find` mode: the skill text is step 3 of +> `skills/link-pr/SKILL.md` and the scanner is `skills/link-pr/scripts/find_sessions.py`. The +> trigger phrases below now sit in link-pr's description. > **Superseded in part (v0.78.0, ENG-1128)** by `docs/specs/find-contributing-sessions-unlinked-prs.md`, > which owns the skill and `find_sessions.py`: no argument now maps the caller's own unlinked PRs @@ -930,7 +945,7 @@ caps are honoured on a synthetic 50k-record session. - **`README.md`**, the `/memhub:*` list — two entries, in the house voice, next to `pr-babysit`: - `/memhub:link-pr [pr] [--session ] [--unlink]` — links a coding session to a pull request in MemHub, so the PR's session context is published from a confirmed fact. - - `/memhub:find-contributing-sessions [pr]` — scans this machine's session history for the + - `/memhub:link-pr --find [pr]` (was `/memhub:find-contributing-sessions [pr]`) — scans this machine's session history for the sessions that wrote a PR's code, ranks the candidates, and links the ones you approve. - **`README.md`**, the PR section — a short paragraph: after a call that addresses GitHub (`gh pr …`, a `curl` / `gh api` request to the REST API, or a GitHub MCP tool), MemHub checks whether diff --git a/scripts/gen_codex_hooks.py b/scripts/gen_codex_hooks.py index 52392892..bd8378e9 100644 --- a/scripts/gen_codex_hooks.py +++ b/scripts/gen_codex_hooks.py @@ -56,8 +56,12 @@ # compatibility bridge in references/codex-hooks-bridge.json is a file the user # already trusted in ~/.codex/hooks.json, and widening it would cost a # re-trust for a detection they can do with /memhub:link-pr. +# `mcp__.+__link_pr` is MemHub's own tool: a `pr_type="fix"` link earns the +# bug-origin instruction (MemHub-Backend pr-weight-success-spec.md §7.1); +# `pr_link.is_memhub_link_pr` is its precise gate. _POST_TOOLS = ("^(Edit|MultiEdit|Write|NotebookEdit|apply_patch|Bash|shell|" - "local_shell|exec_command|shell_command|mcp__.*[Gg]it[Hh]ub.*__.*)$") + "local_shell|exec_command|shell_command|mcp__.*[Gg]it[Hh]ub.*__.*|" + "mcp__.+__link_pr)$") # Claude-only capture scripts: they read Claude's transcript store and must diff --git a/tests/agent_fixture.py b/tests/agent_fixture.py new file mode 100644 index 00000000..fe8cc74e --- /dev/null +++ b/tests/agent_fixture.py @@ -0,0 +1,318 @@ +"""Synthetic Claude Code session trees for the agent-capture suites. + +Not a suite itself (no ``_test`` suffix). ``mirror_9b073366`` builds a tree +shaped like the real session 9b073366 (MemHub-Backend agent-sessions spec +§10 acceptance): 10 Agent-tool agents ending in ``SubagentHandback``, 5 +MemHub harness forks, and 3 dynamic-workflow runs with 11, 46 and 7 agents, +one of them resumed (16 agents of an earlier attempt: 11 done, 5 stopped). +The other builders add the shapes that session does not have. +""" +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from harness_stop import FORK_MARK + +SESSION = '9b073366-0000-4000-8000-000000000001' +SECRET = 'mhk_' + 'A' * 40 + +T0 = datetime(2026, 10, 14, 19, 0, 0, tzinfo=timezone.utc) + + +def iso(dt: datetime) -> str: + return dt.strftime('%Y-%m-%dT%H:%M:%S.') + f'{dt.microsecond // 1000:03d}Z' + + +def ms(dt: datetime) -> int: + return int(dt.timestamp() * 1000) + + +def write_jsonl(path: Path, records) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(''.join(json.dumps(r) + '\n' for r in records)) + + +def user(text_or_blocks, uuid, at=None): + record = {'type': 'user', 'uuid': uuid, 'sessionId': SESSION, + 'message': {'role': 'user', 'content': text_or_blocks}} + if at is not None: + record['timestamp'] = iso(at) + return record + + +def assistant(request_id, blocks, uuid, at=None, stop_reason=None, model='claude-opus-5-5'): + record = {'type': 'assistant', 'uuid': uuid, 'requestId': request_id, + 'message': {'role': 'assistant', 'id': f'msg_{request_id}', 'model': model, + 'content': blocks, 'stop_reason': stop_reason}} + if at is not None: + record['timestamp'] = iso(at) + return record + + +def tool_use(tool_use_id, name, **input_): + return {'type': 'tool_use', 'id': tool_use_id, 'name': name, 'input': input_} + + +def tool_result(tool_use_id, text, is_error=False): + block = {'type': 'tool_result', 'tool_use_id': tool_use_id, + 'content': [{'type': 'text', 'text': text}]} + if is_error: + block['is_error'] = True + return block + + +class Tree: + """A project directory holding one session and everything under it.""" + + def __init__(self, root: Path, session: str = SESSION): + self.session = session + self.root = root + self.transcript = root / f'{session}.jsonl' + self.dir = root / session + self.sub = self.dir / 'subagents' + self.sub.mkdir(parents=True, exist_ok=True) + self.records: list[dict] = [] + + def write_session(self): + write_jsonl(self.transcript, self.records) + + def agent(self, agent_id, records, run_id=None, **meta): + base = self.sub / 'workflows' / run_id if run_id else self.sub + write_jsonl(base / f'agent-{agent_id}.jsonl', records) + (base / f'agent-{agent_id}.meta.json').write_text(json.dumps(meta)) + + def journal(self, run_id, events): + write_jsonl(self.sub / 'workflows' / run_id / 'journal.jsonl', + [{'type': 'launched'}] + events) + + def run_record(self, run_id, record): + path = self.dir / 'workflows' / f'{run_id}.json' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record)) + + def script(self, name, run_id, phases): + path = self.dir / 'workflows' / 'scripts' / f'{name}-{run_id}.js' + path.parent.mkdir(parents=True, exist_ok=True) + rows = ''.join(f" {{ title: '{t}', detail: 'about {t}' }},\n" for t in phases) + path.write_text(f"export const meta = {{\n name: '{name}',\n phases: [\n{rows} ],\n}}\n\n" + "const X = `title: 'Not a phase'`\n") + + +# ── the agents ──────────────────────────────────────────────────────────── + + +def tool_agent_records(agent_id, at, prompt='Audit the db operations'): + return [ + user(prompt, f'{agent_id}-u1', at), + assistant(f'req_{agent_id}_1', [tool_use(f'toolu_{agent_id}_b', 'Bash', command='ls')], + f'{agent_id}-a1', at + timedelta(seconds=1), 'tool_use'), + user([tool_result(f'toolu_{agent_id}_b', 'file')], f'{agent_id}-u2', at + timedelta(seconds=2)), + assistant(f'req_{agent_id}_2', [{'type': 'thinking', 'thinking': ''}], + f'{agent_id}-a2', at + timedelta(seconds=3)), + assistant(f'req_{agent_id}_2', + [tool_use(f'toolu_{agent_id}_h', 'SubagentHandback', + message=f'Findings of {agent_id}: ' + 'x' * 2500)], + f'{agent_id}-a3', at + timedelta(seconds=4)), + ] + + +RELAY = ('[Workflow harness — user request] The harness relays, verbatim and indented below, ' + 'the user request.\n do the thing') +COMPUTED = ('[Workflow harness — computed task] The task text below was computed at runtime.' + ' The computed task text follows:\n \n ## Task\n Do {label}.') + + +def workflow_agent_records(agent_id, label, at, done=True): + records = [ + user(RELAY, f'{agent_id}-u1', at), + user(COMPUTED.format(label=label), f'{agent_id}-u2', at), + assistant(f'req_{agent_id}_1', [tool_use(f'toolu_{agent_id}_r', 'Read', file_path='a')], + f'{agent_id}-a1', at + timedelta(seconds=1), 'tool_use'), + user([tool_result(f'toolu_{agent_id}_r', 'ok')], f'{agent_id}-u3', at + timedelta(seconds=2)), + ] + if done: + records.append(assistant( + f'req_{agent_id}_2', + [tool_use(f'toolu_{agent_id}_s', 'StructuredOutput', answer=f'{label} done')], + f'{agent_id}-a2', at + timedelta(seconds=3))) + else: + records.append(assistant(f'req_{agent_id}_2', [{'type': 'thinking', 'thinking': ''}], + f'{agent_id}-a2', at + timedelta(seconds=3))) + return records + + +def fork_records(agent_id, owner_request, spawn_id, at, owner_blocks=None): + """A fork as Claude Code writes it: a fork-context-ref, a copy of the + owner's spawning response, then the spawn's result plus the directive.""" + return [ + {'type': 'fork-context-ref', 'agentId': agent_id, 'parentSessionId': SESSION}, + assistant(owner_request, owner_blocks or [tool_use(spawn_id, 'Agent', subagent_type='fork', + description='x', prompt='p')], + f'{agent_id}-copy', at), + user([tool_result(spawn_id, 'Fork started — processing in background'), + {'type': 'text', 'text': '\nYou are a worker fork.'}], + f'{agent_id}-u1', at + timedelta(seconds=1)), + assistant(f'req_{agent_id}_1', [{'type': 'text', 'text': f'{agent_id} finished'}], + f'{agent_id}-a1', at + timedelta(seconds=2), 'end_turn'), + ] + + +# ── the 9b073366 mirror ─────────────────────────────────────────────────── + +TOOL_AGENTS = [f'a10{i:02d}' for i in range(10)] +TOOL_USE_IDS = [f'toolu_T{i}' for i in range(10)] +HARNESS_FORKS = [f'af{i:02d}' for i in range(5)] + +RUN_AUDIT = 'wf_65b3bc6f-33a' +RUN_PERF = 'wf_02c95eb8-149' +RUN_SPLIT = 'wf_d2e94298-cb4' + +LAUNCH_AUDIT = 'toolu_013TP1PbjRCigMsbKUPaCjdY' +LAUNCH_PERF = 'toolu_015UjVe4NghhwXHAsHBAGSeY' +RESUME_PERF = 'toolu_019oXcGkg2DaWicntiHey512' +LAUNCH_SPLIT = 'toolu_01EG4dRCsNyQbMUakn4ueaCs' + +PERF_STOPPED = ['ap90', 'ap91', 'ap92', 'ap93', 'ap94'] + + +def _workflow_call(tree, tool_use_id, run_id, task_id, at, with_path=True): + tree.records.append(assistant(f'req_{tool_use_id}', [tool_use(tool_use_id, 'Workflow', script='…')], + f'{tool_use_id}-a', at, 'tool_use')) + text = f'Workflow launched in background. Task ID: {task_id}\nSummary: s' + if with_path: + text += f'\nTranscript dir: {tree.dir}/subagents/workflows/{run_id}' + tree.records.append(user([tool_result(tool_use_id, text)], f'{tool_use_id}-r', + at + timedelta(seconds=1))) + + +def _run(tree, run_id, name, task_id, start, duration_ms, phases, roster, *, extra=None): + """``roster``: [(agent_id, label, phase_index)], all done, in the record.""" + progress = [{'type': 'workflow_phase', 'index': i + 1, 'title': t} for i, (t, _) in enumerate(phases)] + for n, (aid, label, pidx) in enumerate(roster, 1): + progress.append({'type': 'workflow_agent', 'index': n, 'label': label, 'phaseIndex': pidx, + 'phaseTitle': phases[pidx - 1][0], 'agentId': aid, + 'model': 'claude-opus-5-5[1m]', 'state': 'done', 'startedAt': ms(start), + 'queuedAt': ms(start), 'attempt': 1, 'lastToolName': 'StructuredOutput', + 'lastToolSummary': '~/secret/path', 'promptPreview': 'prompt', + 'resultPreview': 'result', 'tokens': 119183, 'toolCalls': 20, + 'durationMs': 396520}) + record = {'runId': run_id, 'taskId': task_id, 'workflowName': name, 'status': 'completed', + 'startTime': ms(start), 'durationMs': duration_ms, 'agentCount': len(roster), + 'script': 'export const meta = {}', 'scriptPath': '/x.js', 'logs': ['log'], + 'summary': f'Summary of {name} with key {SECRET}', 'result': {'pr_url': 'https://x/1'}, + 'phases': [{'title': t, 'detail': d} for t, d in phases], + 'defaultModel': 'claude-opus-5-5[1m]', 'workflowProgress': progress, + 'totalTokens': 959995, 'totalToolCalls': 251} + if extra: + record.update(extra) + tree.run_record(run_id, record) + for aid, label, pidx in roster: + tree.agent(aid, workflow_agent_records(aid, label, start + timedelta(seconds=5)), run_id, + agentType='workflow-subagent', description=label, + workflowPhase=phases[pidx - 1][0], spawnDepth=1, requestShape='foreground') + + +def mirror_9b073366(root: Path) -> Tree: + tree = Tree(root) + at = T0 + tree.records.append(user('audit db operations post sessions tab updates.', 'p0', at)) + # 10 Agent-tool agents, each in its own response. + for i, (aid, tid) in enumerate(zip(TOOL_AGENTS, TOOL_USE_IDS)): + t = at + timedelta(minutes=i + 1) + tree.records.append(assistant(f'req_spawn_{i}', [tool_use(tid, 'Agent', subagent_type='general-purpose', + description=f'audit {i}', + prompt=f'Audit area {i}')], + f'spawn-{i}', t, 'tool_use')) + tree.records.append(user([tool_result(tid, 'Async agent launched')], f'spawn-{i}-r', t)) + tree.agent(aid, tool_agent_records(aid, t), agentType='general-purpose', + description=f'audit {i}', toolUseId=tid, spawnDepth=1) + # 5 MemHub harness forks. + for i, aid in enumerate(HARNESS_FORKS): + t = at + timedelta(minutes=20 + i) + tid = f'toolu_H{i}' + tree.records.append(assistant(f'req_fork_{i}', [tool_use(tid, 'Agent', subagent_type='fork', + description='rule pass', + prompt=f'{FORK_MARK}, moment {SESSION}#{i}')], + f'fork-{i}', t, 'tool_use')) + tree.records.append(user([tool_result(tid, 'Fork started')], f'fork-{i}-r', t)) + tree.agent(aid, fork_records(aid, f'req_fork_{i}', tid, t), agentType='fork', + description='Review the diff', toolUseId=tid, spawnDepth=1) + + # Run 1: 4 phases, 11 agents (3/4/3/1). + s1 = at + timedelta(minutes=45) + _workflow_call(tree, LAUNCH_AUDIT, RUN_AUDIT, 'w2rpn14vi', s1) + phases1 = [('Map', 'map it'), ('Audit', 'audit it'), ('Verify', 'verify'), ('Plan', 'plan')] + roster1 = [(f'aa{n:02d}', f'{t.lower()}:{n}', p) for n, (t, p) in + enumerate([(phases1[p - 1][0], p) for p in [1] * 3 + [2] * 4 + [3] * 3 + [4]])] + _run(tree, RUN_AUDIT, 'studio-pages-server-efficiency-audit', 'w2rpn14vi', s1, 975534, + phases1, roster1) + tree.journal(RUN_AUDIT, [e for aid, label, p in roster1 for e in ( + {'type': 'started', 'agentId': aid, 'label': label, 'phase': phases1[p - 1][0]}, + {'type': 'result', 'agentId': aid, 'result': {}})]) + + # Run 2: launched, then resumed; the record describes the resumed attempt. + s2a = at + timedelta(hours=3) + s2 = at + timedelta(hours=3, minutes=50) + _workflow_call(tree, LAUNCH_PERF, RUN_PERF, 'w7gjcrdzr', s2a) + _workflow_call(tree, RESUME_PERF, RUN_PERF, 'we6rbk483', s2 - timedelta(seconds=1), + with_path=False) + phases2 = [('Implement', 'impl'), ('Review', 'review'), ('Ship', 'ship')] + roster2 = [(f'ap{n:02d}', f'{phases2[p - 1][0].lower()}:{n}', p) + for n, p in enumerate([1] * 10 + [2] * 10 + [3] * 10)] + _run(tree, RUN_PERF, 'studio-dashboards-perf-fixes', 'we6rbk483', s2, 4873930, phases2, roster2) + earlier = [(f'ap{n}', f'implement:{n}', 1) for n in range(60, 68)] \ + + [(f'ap{n}', f'review:{n}', 2) for n in range(70, 71)] \ + + [(f'ap{n}', f'ship:{n}', 3) for n in range(80, 82)] + earlier = earlier[:11] + stopped = [(PERF_STOPPED[0], 'implement:s0', 1), (PERF_STOPPED[1], 'review:s1', 2), + (PERF_STOPPED[2], 'review:s2', 2), (PERF_STOPPED[3], 'ship:s3', 3), + (PERF_STOPPED[4], 'ship:s4', 3)] + events = [] + for aid, label, p in earlier + stopped: + done = (aid, label, p) in earlier + tree.agent(aid, workflow_agent_records(aid, label, s2a + timedelta(minutes=1), done=done), + RUN_PERF, agentType='workflow-subagent', description=label, + workflowPhase=phases2[p - 1][0], spawnDepth=1) + events.append({'type': 'started', 'agentId': aid, 'label': label, 'phase': phases2[p - 1][0]}) + if done: + events.append({'type': 'result', 'agentId': aid, 'result': {}}) + for aid, label, p in roster2: + events.append({'type': 'started', 'agentId': aid, 'label': label, 'phase': phases2[p - 1][0]}) + events.append({'type': 'result', 'agentId': aid, 'result': {}}) + tree.journal(RUN_PERF, events) + + # Run 3: 5 phases, 7 agents (2/1/1/2/1). + s3 = at + timedelta(hours=24, minutes=47) + _workflow_call(tree, LAUNCH_SPLIT, RUN_SPLIT, 'wytn8hhue', s3 - timedelta(seconds=1)) + phases3 = [('Design', 'two designs'), ('Judge', 'pick one'), ('Implement', 'build'), + ('Review', 'review'), ('Ship', 'ship')] + labels3 = [('design:A', 1), ('design:B', 1), ('judge', 2), ('implement', 3), + ('review:equivalence', 4), ('review:migration', 4), ('ship', 5)] + roster3 = [(f'as{n:02d}', label, p) for n, (label, p) in enumerate(labels3)] + _run(tree, RUN_SPLIT, 'investment-split-rollup', 'wytn8hhue', s3, 3893645, phases3, roster3) + tree.journal(RUN_SPLIT, [e for aid, label, p in roster3 for e in ( + {'type': 'started', 'agentId': aid, 'label': label, 'phase': phases3[p - 1][0]}, + {'type': 'result', 'agentId': aid, 'result': {}})]) + tree.write_session() + return tree + + +def journal_only_run(tree: Tree, run_id='wf_inprogress-1', launch='toolu_Wlive', + at=T0 + timedelta(hours=30)): + """A run still going: no record yet, a script with 3 phases, phase 1's two + agents done, phase 2 not started.""" + _workflow_call(tree, launch, run_id, 'tlive', at) + tree.script('live-run', run_id, ['Plan', 'Build', 'Ship']) + tree.journal(run_id, [ + {'type': 'started', 'agentId': 'al01', 'label': 'plan:a', 'phase': 'Plan'}, + {'type': 'started', 'agentId': 'al02', 'label': 'plan:b', 'phase': 'Plan'}, + {'type': 'result', 'agentId': 'al01', 'result': {}}, + {'type': 'result', 'agentId': 'al02', 'result': {}}, + ]) + for aid, label in (('al01', 'plan:a'), ('al02', 'plan:b')): + tree.agent(aid, workflow_agent_records(aid, label, at + timedelta(seconds=5)), run_id, + agentType='workflow-subagent', description=label, workflowPhase='Plan', + spawnDepth=1, model='inherit') + tree.write_session() diff --git a/tests/brain_brief_test.py b/tests/brain_brief_test.py index 86087b69..7d8fe555 100644 --- a/tests/brain_brief_test.py +++ b/tests/brain_brief_test.py @@ -118,6 +118,15 @@ def _brief(payload: dict | None = None) -> dict: check("names the room in agent context", "Repo: XTraceAI/agent-plugins" in _ctx(first)) check("states it is the default target", "DEFAULT target" in _ctx(first)) +check("no session id → no save_artifact session line", "session_id" not in _ctx(first)) +os.environ.pop("CODEX_THREAD_ID", None) +sid_ctx = _ctx(_brief({"cwd": "/repo", "session_id": "abcd-1234"})) +check("names this session for save_artifact, bare on Claude Code", + 'pass `session_id: "abcd-1234"`' in sid_ctx) +sid_ctx = _ctx(_brief({"cwd": "/repo", "session_id": "abcd-1234", + "transcript_path": "/u/.codex/sessions/2026/10/16/rollout-x.jsonl"})) +check("…and codex- on Codex, the id capture sends", + 'pass `session_id: "codex-abcd-1234"`' in sid_ctx) check("first resolution reaches the USER", "systemMessage" in first) second = _brief() @@ -525,10 +534,10 @@ def _prompt(payload: dict) -> dict: brain_brief._write_json(PCACHE, POINTERS) # a budget cut at SessionStart leaves the marker alone, so the prompt delivers the rest. -# 550 is picked to fit some but not all pointers under the head sentence -# (a brief of ~850-900 chars does: 550 tokens × 4 × 2/5 = 880); re-pick it if +# 665 is picked to fit some but not all pointers under the head sentence +# (a brief of ~1050-1090 chars does: 665 tokens × 4 × 2/5 = 1064); re-pick it if # the head grows or the split moves. -os.environ["MEMHUB_BRIEF_TOKEN_BUDGET"] = "550" +os.environ["MEMHUB_BRIEF_TOKEN_BUDGET"] = "665" bctx = _ctx(_brief({"cwd": "/repo", "session_id": "s10"})) shown_at_start = set(brain_brief._ids_in(bctx)) check("the trimmed brief showed some but not all cached pointers", diff --git a/tests/claude_hook_guard_test.py b/tests/claude_hook_guard_test.py index 6c0e16d7..efe760de 100644 --- a/tests/claude_hook_guard_test.py +++ b/tests/claude_hook_guard_test.py @@ -80,7 +80,15 @@ def test_every_claude_handler_is_guarded_and_only_boundaries_capture(): for group in groups: for handler in group["hooks"]: commands.append((event, handler["command"])) - assert len(commands) == 22 # + PostToolUseFailure (rulebook_hook.py post — + assert len(commands) == 26 # + UserPromptSubmit (compare_session.py + # checkpoint — local rewind's before-prompt + # snapshot, ENG-1296), + # + Stop and UserPromptSubmit (rulebook_reply — + # the `reply` rules' gate at Stop, and the + # advisory it leaves for the next prompt), + # + Stop (rulebook_quarantine.py stop — a + # quarantined file blocks the stop once), + # + PostToolUseFailure (rulebook_hook.py post — # the failure lane, ENG-1236), # + UserPromptExpansion (rulebook_hook.py # expansion — a typed command noted, ENG-1235), diff --git a/tests/codex_correctness_test.py b/tests/codex_correctness_test.py index 010eea40..840c0156 100644 --- a/tests/codex_correctness_test.py +++ b/tests/codex_correctness_test.py @@ -20,6 +20,7 @@ import io import json import os +import subprocess import sys import tempfile import time @@ -364,11 +365,30 @@ def test_the_stop_lane_tells_the_hook_it_is_codex(): # The flush lane used to only SHIP rows that already carried their host, so # it needed none — #240 made it also CREATE events, which is what turned a # correct omission into a bug. - with patch.object(bridge, "_detach_flush"), patch.object(bridge, "_run") as run: + quiet = subprocess.CompletedProcess([], 0, b"", b"") + with patch.object(bridge, "_detach_flush"), patch.object(bridge, "_detach_script") as sensor, \ + patch.object(bridge, "_run", return_value=quiet) as run: bridge._dispatch(Path("/plugin"), b"{}", "Stop") - args = run.call_args[0] - assert "rulebook_hook.py" in args, args - assert args[-2:] == ("--host", "codex"), args + # Every Stop lane names the host, including the detached harness sensor. + # Synchronous policy calls retain reply-before-flush ordering. + calls = [c[0] for c in run.call_args_list] + assert calls[0][1:3] == ("rulebook_hook.py", b"{}") and calls[0][3] == "reply", calls + assert sorted(a[1] for a in calls[1:]) == ["rulebook_hook.py", "rulebook_quarantine.py"], calls + sensor.assert_called_once_with(Path("/plugin"), b"{}", "harness_stop.py", "stop", "--host", "codex") + for args in calls + [sensor.call_args[0]]: + assert args[-2:] == ("--host", "codex"), args + + +def test_the_stop_runs_the_reply_rules_before_the_flush(): + # The `reply` rules (rulebook_reply.py) run on Codex's Stop too, as codex, + # and BEFORE the flush, so a fire they record ships with this Stop. + with patch.object(bridge, "_detach_flush"), patch.object(bridge, "_detach_script"), \ + patch.object(bridge, "_run") as run: + bridge._dispatch(Path("/plugin"), b"{}", "Stop") + calls = [c[0] for c in run.call_args_list] + modes = [a[a.index("rulebook_hook.py") + 2] for a in calls if "rulebook_hook.py" in a] + assert modes == ["reply", "flush"], calls + assert calls[0][-2:] == ("--host", "codex"), calls def test_every_bridge_lane_that_writes_a_row_names_its_host(): diff --git a/tests/codex_hooks_parity_test.py b/tests/codex_hooks_parity_test.py index bb7a8628..2d495917 100644 --- a/tests/codex_hooks_parity_test.py +++ b/tests/codex_hooks_parity_test.py @@ -8,6 +8,7 @@ Run: python3 codex_hooks_parity_test.py (stdlib only) """ +import io import json import sys from pathlib import Path @@ -42,14 +43,20 @@ def test_no_claude_only_capture_leaked(): def test_user_bridge_covers_live_codex_capabilities(): + """codex-cli 0.163 ships hook schemas for UserPromptSubmit (input `prompt`, + output additionalContext + systemMessage) and SessionEnd, read from the + binary. Without the first, `prompt` rules, prompt-armed ordering and a + reply advisory's delivery never run on Codex; without the second the last + turn's fires never get their `session_end`.""" bridge = json.loads(BRIDGE_HOOKS.read_text(encoding="utf-8"))["hooks"] - assert set(bridge) == {"SessionStart", "PreToolUse", "PostToolUse", "Stop"} + events = {"SessionStart", "UserPromptSubmit", "PreToolUse", "PostToolUse", + "Stop", "SessionEnd"} + assert set(bridge) == events assert all(len(groups) == 1 for groups in bridge.values()) text = json.dumps(bridge) - for action in ("dispatch SessionStart", "dispatch PreToolUse", - "dispatch PostToolUse", "dispatch Stop"): - assert action in text, action - assert text.count("memhub_hook_bridge.py") == 8 # Unix + Windows, 4 handlers + for event in events: + assert "dispatch " + event in text, event + assert text.count("memhub_hook_bridge.py") == 12 # Unix + Windows, 6 handlers print("PASS test_user_bridge_covers_live_codex_capabilities") @@ -115,6 +122,43 @@ def test_the_pr_link_prefilter_spares_ordinary_shell_calls(): print("PASS test_the_pr_link_prefilter_spares_ordinary_shell_calls") +def test_memhub_link_pr_reaches_the_pr_link_trigger(): + """A `link_pr` call is the bug-origin trigger (MemHub-Backend + pr-weight-success-spec.md §7.1): the bundled matcher must let it in and + the bridge must route it to pr_link_trigger — `unlink_pr` must not.""" + import re # noqa: PLC0415 + sys.path.insert(0, str(ROOT / "plugins" / "memhub" / "scripts")) + import codex_hook_bridge # noqa: PLC0415 + + matcher = re.compile(json.loads(CODEX_HOOKS.read_text(encoding="utf-8")) + ["hooks"]["PostToolUse"][0]["matcher"]) + claude = json.loads(CLAUDE_HOOKS.read_text(encoding="utf-8"))["hooks"]["PostToolUse"] + (claude_matcher,) = [re.compile(g["matcher"]) for g in claude + if any(h["command"].endswith(" pr_link_trigger") for h in g["hooks"])] + link = "mcp__plugin_memhub_memhub__link_pr" + unlink = "mcp__plugin_memhub_memhub__unlink_pr" + for rx in (matcher, claude_matcher): + assert rx.match(link) and rx.match("mcp__memhub__link_pr"), rx.pattern + assert not rx.match(unlink), rx.pattern + + calls = [] + original_link = codex_hook_bridge._pr_link_result + original_rulebook = codex_hook_bridge._rulebook_result + original_stdout = sys.stdout + codex_hook_bridge._pr_link_result = lambda *_a, **_k: calls.append("link") + codex_hook_bridge._rulebook_result = lambda *_a, **_k: None + sys.stdout = io.StringIO() + try: + for tool in (link, unlink, "mcp__memhub__search_memory"): + codex_hook_bridge._dispatch_post(Path("/unused"), b"{}", {"tool_name": tool}) + finally: + sys.stdout = original_stdout + codex_hook_bridge._pr_link_result = original_link + codex_hook_bridge._rulebook_result = original_rulebook + assert calls == ["link"], calls + print("PASS test_memhub_link_pr_reaches_the_pr_link_trigger") + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): diff --git a/tests/codex_hooks_setup_test.py b/tests/codex_hooks_setup_test.py index efa4d5fe..65e84bc0 100644 --- a/tests/codex_hooks_setup_test.py +++ b/tests/codex_hooks_setup_test.py @@ -10,7 +10,9 @@ import subprocess import sys import tempfile -from unittest.mock import patch +import time +import types +from unittest.mock import call, patch from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -46,7 +48,7 @@ def test_install_preserves_other_hooks_and_is_idempotent(): } (home / "hooks.json").write_text(json.dumps(original), encoding="utf-8") changed, expected, backup = setup.install(home) - assert changed and expected == 4 and backup and backup.exists() + assert changed and expected == 6 and backup and backup.exists() installed = json.loads((home / "hooks.json").read_text(encoding="utf-8")) assert installed["description"] == "mine" # The user's own SessionStart hook keeps its slot; ours is appended. @@ -65,10 +67,10 @@ def test_install_preserves_other_hooks_and_is_idempotent(): ] assert any(str(home) in command for command in commands) assert "${CODEX_HOME" not in installed_text - assert setup.status(home)[:3] == (True, 4, 4) + assert setup.status(home)[:3] == (True, 6, 6) again, count, second_backup = setup.install(home) - assert not again and count == 4 and second_backup is None + assert not again and count == 6 and second_backup is None assert json.loads((home / "hooks.json").read_text(encoding="utf-8")) == installed print("PASS test_install_preserves_other_hooks_and_is_idempotent") @@ -121,7 +123,7 @@ def test_install_replaces_legacy_bridge_without_duplicates(): assert "plugins/cache/xtrace-plugins" not in text assert "0.26/scripts/codex_flush.py" not in text assert "echo user" in text - assert setup.status(home)[:3] == (True, 4, 4) + assert setup.status(home)[:3] == (True, 6, 6) print("PASS test_install_replaces_legacy_bridge_without_duplicates") @@ -466,7 +468,7 @@ def test_status_checks_materialized_windows_commands(): doc["hooks"]["Stop"][0]["hooks"][0]["commandWindows"] = "py -3 broken.py" (home / "hooks.json").write_text(json.dumps(doc), encoding="utf-8") healthy, actual, expected, _hooks_ok, _runner_ok = setup.status(home) - assert not healthy and actual == expected == 4 + assert not healthy and actual == expected == 6 print("PASS test_status_checks_materialized_windows_commands") @@ -482,10 +484,10 @@ def test_cli_names_only_the_memhub_handlers_for_review(): sys.argv = real_argv text = output.getvalue() - assert "installed (4 handlers)" in text + assert "installed (6 handlers)" in text assert f"User config - {Path(raw) / 'hooks.json'}" in text assert f"review command: {Path(raw) / 'memhub_hook_bridge.py'}" in text - assert "trust only the 4 handlers" in text + assert "trust only the 6 handlers" in text assert "do not use 'Trust all'" in text print("PASS test_cli_names_only_the_memhub_handlers_for_review") @@ -518,11 +520,130 @@ def test_rulebook_denial_survives_context_merge(): def test_stop_flushes_rule_fires_and_capture(): - with patch.object(bridge, "_detach_flush") as capture, patch.object(bridge, "_run") as fire_flush: + quiet = subprocess.CompletedProcess([], 0, b"", b"") + with patch.object(bridge, "_detach_flush") as capture, \ + patch.object(bridge, "_detach_script") as detach, \ + patch.object(bridge, "_run", return_value=quiet) as run: bridge._dispatch(Path("/plugin"), b"{}", "Stop") capture.assert_called_once_with(Path("/plugin"), b"{}", "Stop") - fire_flush.assert_called_once_with(Path("/plugin"), "rulebook_hook.py", b"{}", - "flush", "final", "--host", "codex", timeout=7) + # The reply rules first (rulebook_reply.py); then, side by side and so in + # no fixed order, the fire flush and quarantine check. Classification is + # detached and contributes no Stop output. + assert run.call_count == 3, run.call_args_list + detach.assert_called_once_with(Path("/plugin"), b"{}", "harness_stop.py", "stop", "--host", "codex") + assert run.call_args_list[0] == call(Path("/plugin"), "rulebook_hook.py", b"{}", + "reply", "--host", "codex", timeout=7) + rest = run.call_args_list[1:] + for expected in (call(Path("/plugin"), "rulebook_hook.py", b"{}", + "flush", "final", "--host", "codex", timeout=7), + call(Path("/plugin"), "rulebook_quarantine.py", b"{}", + "stop", "--host", "codex", timeout=7)): + assert expected in rest, rest + + +def _stop_out(answers): + """The bridge's Stop output when each lane answers `answers[]` + (`reply`, `flush`, `quarantine`), quiet otherwise. Keyed by lane, not by + call order: the flush and quarantine run side by side.""" + quiet = subprocess.CompletedProcess([], 0, b"", b"") + + def run(root, script, payload, *args, timeout=7): + lane = {"rulebook_quarantine.py": "quarantine", "harness_stop.py": "harness"}.get( + script, args[0] if args else "") + return answers.get(lane, quiet) + out = io.BytesIO() + fake = types.SimpleNamespace(buffer=out, write=lambda s: out.write(s.encode())) + with patch.object(bridge, "_detach_flush"), patch.object(bridge, "_detach_script"), \ + patch.object(bridge, "_run", side_effect=run), patch.object(bridge.sys, "stdout", fake): + bridge._dispatch(Path("/plugin"), b"{}", "Stop") + return out.getvalue() + + +def _status_text(home: Path, state_dir: Path) -> str: + real_argv = sys.argv + sys.argv = ["setup_codex_hooks.py", "status", "--codex-home", str(home)] + output = io.StringIO() + try: + with patch.object(setup, "_HOOK_STATE_DIR", state_dir), \ + contextlib.redirect_stdout(output): + assert setup.main() == 0 + finally: + sys.argv = real_argv + return output.getvalue() + + +def _age(path: Path, seconds_ago: float) -> None: + stamp = time.time() - seconds_ago + os.utime(path, (stamp, stamp)) + + +def test_status_confirms_trust_only_from_state_newer_than_the_install(): + """Codex runs a user hook only once it is trusted, so bridge-written state + after the install is evidence; anything else stays unverified.""" + with tempfile.TemporaryDirectory() as raw: + home, state = Path(raw) / "codex", Path(raw) / "codexflush" + home.mkdir() + setup.install(home) + _age(home / "hooks.json", 600) + _age(home / "memhub_hook_bridge.py", 600) + + # No state directory at all: a fresh install nobody has trusted yet. + text = _status_text(home, state) + assert "trust: Codex-controlled; verify the 6 MemHub handlers" in text, text + assert "confirmed" not in text, text + + # State from before the install (an older bridge) proves nothing. + state.mkdir() + old = state / "codex-old.json" + old.write_text("{}", encoding="utf-8") + _age(old, 3600) + assert setup.hooks_ran_at(home, state) is None + assert "confirmed" not in _status_text(home, state) + + # A capture write after the install: the hooks ran, so trust is real. + fresh = state / "codex-new.json" + fresh.write_text("{}", encoding="utf-8") + ran = setup.hooks_ran_at(home, state) + assert ran is not None and ran == fresh.stat().st_mtime + text = _status_text(home, state) + assert "trust: confirmed (MemHub hooks ran at " in text, text + assert "Codex-controlled" not in text, text + + # Re-installing a changed bridge asks for fresh evidence again. + doc = json.loads((home / "hooks.json").read_text(encoding="utf-8")) + doc["hooks"]["Stop"][0]["hooks"][0]["timeout"] = 31 + (home / "hooks.json").write_text(json.dumps(doc), encoding="utf-8") + stamp = time.time() + 5 + os.utime(home / "hooks.json", (stamp, stamp)) + assert setup.hooks_ran_at(home, state) is None + print("PASS test_status_confirms_trust_only_from_state_newer_than_the_install") + + +def test_trust_is_never_reported_for_an_unhealthy_install(): + with tempfile.TemporaryDirectory() as raw: + home, state = Path(raw) / "codex", Path(raw) / "codexflush" + state.mkdir(parents=True) + (state / "codex-new.json").write_text("{}", encoding="utf-8") + home.mkdir() + real_argv = sys.argv + sys.argv = ["setup_codex_hooks.py", "status", "--codex-home", str(home)] + output = io.StringIO() + try: + with patch.object(setup, "_HOOK_STATE_DIR", state), \ + contextlib.redirect_stdout(output): + assert setup.main() == 1 + finally: + sys.argv = real_argv + assert "NOT INSTALLED" in output.getvalue() + assert "trust:" not in output.getvalue() + assert setup.hooks_ran_at(home, state) is None + print("PASS test_trust_is_never_reported_for_an_unhealthy_install") + + +def test_hook_state_dir_matches_the_writers(): + """The evidence directory is the one the bridge's lanes write.""" + assert setup._HOOK_STATE_DIR == bridge._STATE_DIR == _load("codex_flush").STATE_DIR + print("PASS test_hook_state_dir_matches_the_writers") if __name__ == "__main__": diff --git a/tests/cursor_capture_test.py b/tests/cursor_capture_test.py index e5205b5c..8202d9f7 100644 --- a/tests/cursor_capture_test.py +++ b/tests/cursor_capture_test.py @@ -61,6 +61,30 @@ def test_cross_platform_launcher_acknowledges_and_detaches(): print("PASS test_cross_platform_launcher_acknowledges_and_detaches") +def test_session_start_reaches_the_flusher_for_its_clock(): + """sessionStart is a rulebook event, but its clock dates the session's + first records, so the flusher gets it too; other rulebook events don't.""" + seen: list[str] = [] + original = (cursor_capture.spawn_cursor_flush, cursor_capture.cursor_rulebook.note, + sys.stdin, sys.stdout, sys.argv) + try: + cursor_capture.spawn_cursor_flush = lambda raw, event: seen.append(event) + cursor_capture.cursor_rulebook.note = lambda *_args, **_kwargs: None + outputs = [] + for event in ("sessionStart", "preToolUse", "postToolUse"): + sys.stdin = io.TextIOWrapper(io.BytesIO(b'{"session_id":"s"}')) + sys.stdout = io.StringIO() + sys.argv = ["cursor_capture.py", event] + assert cursor_capture.main() == 0 + outputs.append(json.loads(sys.stdout.getvalue())) + finally: + (cursor_capture.spawn_cursor_flush, cursor_capture.cursor_rulebook.note, + sys.stdin, sys.stdout, sys.argv) = original + assert seen == ["sessionStart"] + assert outputs[0] == {} # sessionStart still takes no permission + print("PASS test_session_start_reaches_the_flusher_for_its_clock") + + def test_detached_temporary_stdin_survives_parent_close(): """The detached child must retain stdin after the launcher returns. @@ -120,11 +144,18 @@ def test_cursor_manifest_uses_one_portable_launcher_per_event(): "beforeShellExecution", "afterShellExecution", "afterFileEdit", "afterAgentResponse", "stop", "beforeSubmitPrompt", "sessionEnd", # The rulebook lane (cursor_rulebook.py). - "sessionStart", "preToolUse", "postToolUse", "beforeReadFile", + "sessionStart", "preToolUse", "postToolUse", "postToolUseFailure", + "beforeReadFile", + # Sub-agents (cursor_agents.py): subagentStop names the child; Start + # is answered and ignored. Neither fires in cursor-agent. + "subagentStart", "subagentStop", } - # Only the Read/Task calls the rulebook lane tracks, not every tool call. - assert document["hooks"]["preToolUse"][0]["matcher"] == "Read|Task" - assert document["hooks"]["postToolUse"][0]["matcher"] == "Task" + # Only the calls the rulebook lanes read, not every tool call: Read/Task + # are tracked, edits are gated and post-read, Shell results post-read. + edits = "Write|Edit|StrReplace|MultiEdit|search_replace" + assert document["hooks"]["preToolUse"][0]["matcher"] == f"Read|Task|{edits}" + assert document["hooks"]["postToolUse"][0]["matcher"] == f"Task|Shell|{edits}" + assert document["hooks"]["postToolUseFailure"][0]["matcher"] == "Shell" for event, handlers in document["hooks"].items(): assert len(handlers) == 1, event command = handlers[0]["command"] diff --git a/tests/cursor_timestamp_test.py b/tests/cursor_timestamp_test.py index 965ea913..f93be060 100644 --- a/tests/cursor_timestamp_test.py +++ b/tests/cursor_timestamp_test.py @@ -186,6 +186,7 @@ def _payload(path: Path, event: str, generation: str, text: str) -> dict: "transcript_path": str(path), "workspace_roots": [str(path.parent)], "model": "cursor-test-model", + # IDE input is gross of its 5 cache reads, so 95 is stored. "input_tokens": 100, "output_tokens": 10, "cache_read_tokens": 5, "cache_write_tokens": 1, "status": "completed", @@ -256,7 +257,7 @@ async def fake_flush(uuid, source_path, blob_ids, flush_mode="now", # backlog; the mid-turn record has no real clock and stays # unmeasured. assert _stamps_of(calls[0]) == [ISO_1, ISO_1, None, NOW_1] - assert calls[0][3]["message"]["usage"]["input_tokens"] == 100 + assert calls[0][3]["message"]["usage"]["input_tokens"] == 95 state = cursor_flush._read_state(SESSION) assert sorted(state["record_ts"].values(), key=lambda v: (v is None, v)) == [ @@ -272,7 +273,7 @@ async def fake_flush(uuid, source_path, blob_ids, flush_mode="now", assert len(calls) == 2 assert _stamps_of(calls[1]) == [ ISO_1, ISO_1, None, NOW_1, ISO_2, NOW_2, NOW_2] - assert calls[1][6]["message"]["usage"]["input_tokens"] == 100 + assert calls[1][6]["message"]["usage"]["input_tokens"] == 95 assert _stamps_of(calls[1])[:4] == _stamps_of(calls[0]) # A third boundary with NO new content but a fresh usage sample: @@ -348,8 +349,8 @@ async def fake_flush(uuid, source_path, blob_ids, flush_mode="now", ISO_1, ISO_1, None, None, ISO_2, None, None, None] cursor_flush.apply_session_state(records, SESSION) assert _stamps_of(records) == _stamps_of(calls[3]) - assert records[3]["message"]["usage"]["input_tokens"] == 100 - assert records[7]["message"]["usage"]["input_tokens"] == 100 + assert records[3]["message"]["usage"]["input_tokens"] == 95 + assert records[7]["message"]["usage"]["input_tokens"] == 95 finally: cursor_flush.STATE_DIR = originals["state_dir"] cursor_flush._CURSOR_PROJECTS = originals["projects"] @@ -412,6 +413,9 @@ def test_event_can_flush_seals_quiet_observers(): assert can(event, {}), event assert not can("someFutureHook", {}) assert not can("unknown", {}) + # sessionStart saves only its clock (see the sessionStart tests below); + # it never reads the transcript, so it stays outside the observers. + assert not can("sessionStart", {}) print("PASS test_event_can_flush_seals_quiet_observers") @@ -472,6 +476,259 @@ def test_apply_session_state_refuses_non_uuid_ids(): print("PASS test_apply_session_state_refuses_non_uuid_ids") +# ---- hook wall clocks: sessionStart, beforeSubmitPrompt, sessionEnd ------ + + +class _Flow: + """main() against a fixture transcript, with the clock and send faked.""" + + def __init__(self, root: Path): + self.path = (root / "projects" / "fixture" / "agent-transcripts" / + SESSION / f"{SESSION}.jsonl") + self.root = root + self.calls: list[list[dict]] = [] + self.now = NOW_1 + + def __enter__(self): + self.originals = (cursor_flush.STATE_DIR, cursor_flush._CURSOR_PROJECTS, + cursor_flush.cursor_reader._CHATS, + cursor_flush.cursor_reader._PROJECTS, + cursor_flush._flush, cursor_flush._log, + cursor_flush._now_iso) + + async def fake_flush(uuid, source_path, blob_ids, flush_mode="now", + **kwargs): + self.calls.append(json.loads(json.dumps(kwargs["records"]))) + cursor_flush._save_state( + uuid, transcript_revision=kwargs["source_revision"], + sent_usage_generations=sorted(kwargs["applied_usage"]), + pending_pr_urls=[], last_flush_at=1_787_000_000.0) + + cursor_flush.STATE_DIR = self.root / "state" + cursor_flush._CURSOR_PROJECTS = self.root / "projects" + cursor_flush.cursor_reader._CHATS = self.root / "chats" + cursor_flush.cursor_reader._PROJECTS = self.root / "projects" + cursor_flush._flush = fake_flush + cursor_flush._log = lambda _message: None + cursor_flush._now_iso = lambda: self.now + return self + + def __exit__(self, *_exc): + (cursor_flush.STATE_DIR, cursor_flush._CURSOR_PROJECTS, + cursor_flush.cursor_reader._CHATS, cursor_flush.cursor_reader._PROJECTS, + cursor_flush._flush, cursor_flush._log, + cursor_flush._now_iso) = self.originals + + def hook(self, event: str, now: str, **extra) -> None: + self.now = now + payload = {"hook_event_name": event, "session_id": SESSION, + "conversation_id": SESSION, "generation_id": "", + "workspace_roots": [str(self.path.parent)], + "model": "cursor-test-model", + # sessionStart's transcript_path is null on IDE and CLI. + "transcript_path": (None if event == "sessionStart" + else str(self.path))} + payload.update(extra) + assert _run_main(event, payload) == 0 + + def state(self) -> dict: + return cursor_flush._read_state(SESSION) + + +START = "2026-08-23T18:16:12.481Z" # inside TAG_1's minute (18:16) +END_1 = "2026-08-23T18:18:20.904Z" # after TAG_2's minute starts (18:18) +END_2 = "2026-08-23T18:18:47.115Z" + + +def test_session_start_saves_its_clock_without_observing(): + with tempfile.TemporaryDirectory() as td, _Flow(Path(td)) as flow: + _write_rows(flow.path, TURN_1) + flow.hook("sessionStart", START) + state = flow.state() + assert state["session_start_at"] == START + # No pin map, no source, no send: the first REAL observation is still + # a first observation, so its backlog semantics are unchanged. + assert "record_ts" not in state and "source_kind" not in state + assert flow.calls == [] + # A repeated sessionStart never moves the session's start later. + flow.hook("sessionStart", "2026-08-23T18:40:00.000Z") + assert flow.state()["session_start_at"] == START + print("PASS test_session_start_saves_its_clock_without_observing") + + +def test_cli_session_end_dates_each_invocation_and_resends_are_stable(): + """cursor-agent: sessionStart once, then one sessionEnd per invocation + and no usage hooks. Before, invocation 1 shipped only minute tags and + undated records — the session spanned 0 s of agent time.""" + with tempfile.TemporaryDirectory() as td, _Flow(Path(td)) as flow: + flow.hook("sessionStart", START) + _write_rows(flow.path, TURN_1) + flow.hook("sessionEnd", END_1, reason="completed") + assert len(flow.calls) == 1 + # Banner and prompt: the session's exact start beats the 18:16 tag. + # Final assistant: the sessionEnd clock, though this is the first + # observation. The mid-turn record has no real clock and stays NULL. + assert _stamps_of(flow.calls[0]) == [START, START, None, END_1] + + # --resume: no sessionStart. Invocation 2's prompt carries only an + # 18:18 tag, which reads EARLIER than invocation 1's end (18:18:20): + # it is raised to that bound rather than reordered before it. + _write_rows(flow.path, TURN_1 + TURN_2) + flow.hook("sessionEnd", END_2, reason="completed") + assert len(flow.calls) == 2 + stamps = _stamps_of(flow.calls[1]) + assert stamps == [START, START, None, END_1, END_1, END_2, END_2] + assert stamps[:4] == _stamps_of(flow.calls[0]) + dated = [cursor_flush._parse_iso(s) for s in stamps if s] + assert dated == sorted(dated) + assert (dated[-1] - dated[0]).total_seconds() > 0 + + # A re-send at a much later wall clock (forced here by queued PR + # evidence) ships the ORIGINAL dates, never its own clock. + flow.hook("afterShellExecution", "2026-08-23T23:00:00.000Z", + command="gh pr create --fill", + output="https://github.com/xtraceai/agent-plugins/pull/9") + assert len(flow.calls) == 3 + assert _stamps_of(flow.calls[2]) == stamps + records, _meta = cursor_reader.to_canonical(flow.path, session_id=SESSION) + cursor_flush.apply_session_state(records, SESSION) + assert _stamps_of(records) == stamps + print("PASS test_cli_session_end_dates_each_invocation_and_resends_are_stable") + + +def test_session_end_dates_its_records_at_the_hook_clock(): + """The sessionEnd boundary may wait on a busy upload before it reads; + the invocation's new records must not then be dated after the final + record its clock dates (seen live: 4 ms out of order).""" + with tempfile.TemporaryDirectory() as td, _Flow(Path(td)) as flow: + flow.hook("sessionStart", START) + _write_rows(flow.path, TURN_1) + flow.hook("sessionEnd", END_1) + _write_rows(flow.path, TURN_1 + TURN_2) + later = iter([END_2, "2026-08-23T18:18:49.000Z"]) + cursor_flush._now_iso = lambda: next(later) + assert _run_main("sessionEnd", { + "hook_event_name": "sessionEnd", "session_id": SESSION, + "conversation_id": SESSION, "transcript_path": str(flow.path), + "workspace_roots": [str(flow.path.parent)]}) == 0 + assert _stamps_of(flow.calls[-1])[-2:] == [END_2, END_2] + print("PASS test_session_end_dates_its_records_at_the_hook_clock") + + +def test_session_end_never_dates_records_written_while_it_waits(): + """An in-flight send can hold the upload lock while the next --resume + invocation writes its turn. sessionEnd observes before that wait, so its + clock dates only its own invocation; the next sessionEnd keeps its end.""" + with tempfile.TemporaryDirectory() as td, _Flow(Path(td)) as flow: + flow.hook("sessionStart", START) + _write_rows(flow.path, TURN_1) + real_acquire = cursor_flush._acquire + + def busy_upload(uuid, blocking=False, *, observations=False): + if not observations: + # Invocation 2 runs while this hook waits on the send. + _write_rows(flow.path, TURN_1 + TURN_2) + flow.now = "2026-08-23T18:18:40.000Z" + return real_acquire(uuid, blocking, observations=observations) + + cursor_flush._acquire = busy_upload + try: + flow.hook("sessionEnd", END_1) + finally: + cursor_flush._acquire = real_acquire + assert _stamps_of(flow.calls[-1]) == [START, START, None, END_1] + flow.hook("sessionEnd", END_2) + stamps = _stamps_of(flow.calls[-1]) + assert stamps == [START, START, None, END_1, END_1, END_2, END_2] + print("PASS test_session_end_never_dates_records_written_while_it_waits") + + +def test_ide_before_submit_prompt_clock_beats_the_minute_tag(): + prompt_1 = "2026-08-23T18:16:31.579Z" + stop_1 = "2026-08-23T18:16:40.250Z" + prompt_2 = "2026-08-23T18:18:02.991Z" + stop_2 = "2026-08-23T18:18:33.425Z" + with tempfile.TemporaryDirectory() as td, _Flow(Path(td)) as flow: + # The IDE fires sessionStart when the chat opens, which can be well + # before the first prompt: 18:15:40 is outside the 18:16 tag's + # minute, so it dates nothing and the banner keeps its tag. + flow.hook("sessionStart", "2026-08-23T18:15:40.000Z") + # beforeSubmitPrompt fires before Cursor writes the prompt (the + # transcript does not exist yet): its clock is queued, not lost. + flow.hook("beforeSubmitPrompt", prompt_1, prompt="Reply once") + assert len(flow.state()["prompt_clocks"]) == 1 + _write_rows(flow.path, TURN_1) + flow.hook("stop", stop_1, generation_id=GEN_1, input_tokens=100, + output_tokens=10, status="completed") + assert _stamps_of(flow.calls[-1]) == [ISO_1, prompt_1, None, stop_1] + assert flow.state()["prompt_clocks"] == [] + + # Turn 2's clock waits through a boundary that sees no new prompt, + # then lands on the prompt whose text it announced. + flow.hook("beforeSubmitPrompt", prompt_2, prompt="Edit it") + _write_rows(flow.path, TURN_1 + TURN_2) + flow.hook("stop", stop_2, generation_id=GEN_2, input_tokens=100, + output_tokens=10, status="completed") + assert _stamps_of(flow.calls[-1]) == [ + ISO_1, prompt_1, None, stop_1, prompt_2, stop_2, stop_2] + assert flow.state()["prompt_clocks"] == [] + print("PASS test_ide_before_submit_prompt_clock_beats_the_minute_tag") + + +def test_prompt_clock_only_dates_its_own_prompt(): + with tempfile.TemporaryDirectory() as td, _Flow(Path(td)) as flow: + # A submit Cursor never wrote (different text) stays queued and + # dates nothing; the prompt keeps its tag. + flow.hook("beforeSubmitPrompt", "2026-08-23T18:16:20.000Z", + prompt="something else") + _write_rows(flow.path, TURN_1) + flow.hook("stop", NOW_1, generation_id=GEN_1, input_tokens=100, + output_tokens=10, status="completed") + assert _stamps_of(flow.calls[-1]) == [ISO_1, ISO_1, None, NOW_1] + assert len(flow.state()["prompt_clocks"]) == 1 + # Unit: a clock that reads earlier than the prompt's own tag minute is + # not its clock; an already-pinned prompt is never re-dated. + records = [_rec("banner", ts=ISO_1, role="user"), { + "type": "user", "uuid": "ask", "timestamp": ISO_2, + "message": {"role": "user", "content": "Edit it"}}] + state = {"prompt_clocks": [{"at": "2026-08-23T18:17:59.000Z", + "digest": cursor_flush._prompt_digest("Edit it")}]} + assert cursor_flush._exact_clocks(records, state, "stop", NOW_2) == ({}, []) + state["record_ts"] = {"ask": ISO_2} + state["prompt_clocks"][0]["at"] = NOW_2 + assert cursor_flush._exact_clocks(records, state, "stop", NOW_2) == ({}, []) + print("PASS test_prompt_clock_only_dates_its_own_prompt") + + +def test_new_prompt_is_never_dated_before_the_previous_turn(): + def ask(uuid, ts): + return {"type": "user", "uuid": uuid, "timestamp": ts, + "message": {"role": "user", "content": uuid}} + late = "2026-08-23T18:18:40.000Z" + records = [ask("p1", ISO_1), _rec("a1"), ask("p2", ISO_2), _rec("a2")] + stamps = cursor_flush._stamp_records( + records, {"p1": ISO_1, "a1": late}, NOW_2, first_observation=False) + # p2's tag (18:18:00) predates a1's pin: raised to it, not reordered. + assert _stamps_of(records) == [ISO_1, late, late, NOW_2] + assert stamps["p2"] == late + # Same for a hook clock; and a later-than-floor clock is left alone. + records = [ask("p1", ISO_1), _rec("a1"), ask("p2", ISO_2)] + cursor_flush._stamp_records( + records, {"p1": ISO_1, "a1": late}, NOW_2, first_observation=False, + clocks={"p2": "2026-08-23T18:18:10.000Z"}) + assert records[2]["timestamp"] == late + records = [ask("p1", ISO_1), _rec("a1"), ask("p2", ISO_2)] + cursor_flush._stamp_records( + records, {"p1": ISO_1, "a1": NOW_1}, NOW_2, first_observation=False, + clocks={"p2": "2026-08-23T18:18:10.000Z"}) + assert records[2]["timestamp"] == "2026-08-23T18:18:10.000Z" + # Read-only replay never re-dates: pins and tags ride as they are. + records = [ask("p1", ISO_1), _rec("a1"), ask("p2", ISO_2)] + cursor_flush._stamp_records(records, {"a1": late}, None, first_observation=True) + assert _stamps_of(records) == [ISO_1, late, ISO_2] + print("PASS test_new_prompt_is_never_dated_before_the_previous_turn") + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): diff --git a/tests/cursor_usage_test.py b/tests/cursor_usage_test.py index 79d090bb..eb446f92 100644 --- a/tests/cursor_usage_test.py +++ b/tests/cursor_usage_test.py @@ -117,8 +117,9 @@ def test_hook_usage_is_exact_and_aborts_remain_unmeasured(): generation, usage = cursor_flush._hook_usage( "afterAgentResponse", payload) assert generation == GENERATION + # The IDE's input_tokens (20120) includes its 1024 cache reads; stored net. assert usage == { - "input_tokens": 20120, + "input_tokens": 19096, "output_tokens": 48, "cache_read_input_tokens": 1024, "cache_creation_input_tokens": 9, @@ -336,12 +337,13 @@ def test_main_persists_usage_before_send_and_deduplicates_stop(): final = calls[0]["records"][-1] assert final["message"]["content"][0]["text"] == "USAGE_OK" assert final["message"]["usage"] == { - "input_tokens": 20120, "output_tokens": 48, + "input_tokens": 19096, "output_tokens": 48, "cache_read_input_tokens": 1024, "cache_creation_input_tokens": 9, } assert calls[0]["applied_usage"] == {GENERATION} assert list(state["usage_events"]) == [GENERATION] + assert state["usage_events"][GENERATION]["basis"] == "net" assert state["source_kind"] == "transcript" assert state["sent_usage_generations"] == [GENERATION] print("PASS test_main_persists_usage_before_send_and_deduplicates_stop") @@ -377,6 +379,186 @@ def test_sticky_transcript_source_never_switches_to_late_store(): print("PASS test_sticky_transcript_source_never_switches_to_late_store") +GROSS = {"input_tokens": 145087, "output_tokens": 295, + "cache_read_tokens": 143616, "cache_write_tokens": 7} +NET = {"input_tokens": 1471, "output_tokens": 295, + "cache_read_input_tokens": 143616, "cache_creation_input_tokens": 7} + + +def _assistant(uuid: str) -> dict: + return {"type": "assistant", "uuid": uuid, + "message": {"role": "assistant", "content": [ + {"type": "text", "text": "DONE"}]}} + + +def test_gross_ide_hook_usage_is_stored_net_of_cache_reads(): + payload = dict(GROSS, generation_id=GENERATION) + generation, usage = cursor_flush._hook_usage("stop", payload) + # Cache writes are left alone: whether IDE input includes them is unverified. + assert (generation, usage) == (GENERATION, NET) + events = cursor_flush._usage_events_with({}, generation, "final", usage) + assert events[GENERATION] == {"target_uuid": "final", "usage": NET, + "basis": "net"} + records = [_assistant("final")] + assert cursor_flush._apply_usage(records, events) == {GENERATION} + assert records[0]["message"]["usage"] == NET # not subtracted a second time + print("PASS test_gross_ide_hook_usage_is_stored_net_of_cache_reads") + + +def test_cache_reads_above_input_clamp_to_zero(): + _, usage = cursor_flush._hook_usage("afterAgentResponse", { + "generation_id": GENERATION, "input_tokens": 10, + "cache_read_tokens": 50, "output_tokens": 1}) + assert usage["input_tokens"] == 0 + assert usage["cache_read_input_tokens"] == 50 + print("PASS test_cache_reads_above_input_clamp_to_zero") + + +def test_legacy_unmarked_usage_is_converted_once(): + legacy_generation = "0f6c2d1e-7b3a-4c5d-9e8f-a1b2c3d4e5f6" + # Strict reads accept only UUID-shaped record ids. + OLD_TURN = "6f1d3b2a-0c4e-5d6f-8a9b-0c1d2e3f4a5b" + NEW_TURN = "7a2e4c3b-1d5f-5e7a-9b0c-1d2e3f4a5b6c" + gross = cursor_flush.cursor_reader.normalize_usage(GROSS) + # Saved by a plugin from before the basis marker: gross, no "basis". + state = {"usage_events": { + legacy_generation: {"target_uuid": OLD_TURN, "usage": gross}}} + with tempfile.TemporaryDirectory() as td: + old_state = cursor_flush.STATE_DIR + cursor_flush.STATE_DIR = Path(td) + try: + cursor_flush._state_path(SESSION).write_text( + json.dumps(state), encoding="utf-8") + assert cursor_flush._read_state(SESSION, strict=True) == state + finally: + cursor_flush.STATE_DIR = old_state + + # A session straddling the upgrade: the new turn is stored net and marked, + # the old one stays as saved and is converted only when applied. + _, usage = cursor_flush._hook_usage( + "stop", dict(GROSS, generation_id=GENERATION)) + events = cursor_flush._usage_events_with(state, GENERATION, NEW_TURN, usage) + assert "basis" not in events[legacy_generation] + assert events[legacy_generation]["usage"] == gross + for _ in range(2): # every read converts the unchanged saved entry once + records = [_assistant(OLD_TURN), _assistant(NEW_TURN)] + assert cursor_flush._apply_usage(records, events) == { + legacy_generation, GENERATION} + assert records[0]["message"]["usage"] == NET + assert records[1]["message"]["usage"] == NET + + # Re-sent after the upgrade, the legacy generation is replaced by a marked + # net sample rather than being subtracted on top of its old value. + resent = cursor_flush._usage_events_with( + {"usage_events": events}, legacy_generation, "elsewhere", usage) + assert resent[legacy_generation] == { + "target_uuid": OLD_TURN, "usage": NET, "basis": "net"} + records = [_assistant(OLD_TURN)] + cursor_flush._apply_usage(records, resent) + assert records[0]["message"]["usage"] == NET + + # A basis this build does not know is left unmeasured, and refused on a + # strict read, rather than trusted or converted. + unknown = {GENERATION: {"target_uuid": NEW_TURN, "usage": NET, + "basis": "gross"}} + assert cursor_flush._apply_usage([_assistant(NEW_TURN)], unknown) == set() + with tempfile.TemporaryDirectory() as td: + old_state = cursor_flush.STATE_DIR + cursor_flush.STATE_DIR = Path(td) + try: + cursor_flush._state_path(SESSION).write_text( + json.dumps({"usage_events": unknown}), encoding="utf-8") + try: + cursor_flush._read_state(SESSION, strict=True) + except ValueError: + pass + else: + raise AssertionError("an unknown usage basis passed a strict read") + finally: + cursor_flush.STATE_DIR = old_state + print("PASS test_legacy_unmarked_usage_is_converted_once") + + +def test_resent_generation_stays_idempotent(): + payload = dict(GROSS, generation_id=GENERATION) + _, first = cursor_flush._hook_usage("afterAgentResponse", payload) + events = cursor_flush._usage_events_with({}, GENERATION, "final", first) + _, again = cursor_flush._hook_usage("stop", payload) + events = cursor_flush._usage_events_with( + {"usage_events": events}, GENERATION, "final", again) + assert events == {GENERATION: {"target_uuid": "final", "usage": NET, + "basis": "net"}} + records = [_assistant("final")] + cursor_flush._apply_usage(records, events) + cursor_flush._apply_usage(records, events) + assert records[0]["message"]["usage"] == NET + print("PASS test_resent_generation_stays_idempotent") + + +def test_recorded_ide_session_stores_net_usage_per_turn(): + """Replay the usage hooks of a real three-turn Cursor IDE session + (tests/fixtures/cursor_ide_usage_recording.json) through main(), each + against the transcript as it stood when the hook fired.""" + fixture = json.loads((ROOT / "tests" / "fixtures" / + "cursor_ide_usage_recording.json").read_text( + encoding="utf-8")) + session = fixture["session_id"] + with tempfile.TemporaryDirectory() as td: + root = Path(td) + projects = root / "projects" + path = (projects / "recording" / "agent-transcripts" / session / + f"{session}.jsonl") + path.parent.mkdir(parents=True) + sent: list[list[dict]] = [] + originals = (cursor_flush.STATE_DIR, cursor_flush._CURSOR_PROJECTS, + cursor_flush.cursor_reader._CHATS, + cursor_flush.cursor_reader._PROJECTS, + cursor_flush._flush, cursor_flush._log) + + async def fake_flush(uuid, source_path, blob_ids, flush_mode="now", + **kwargs): + sent.append(json.loads(json.dumps(kwargs["records"]))) + cursor_flush._save_state( + uuid, transcript_revision=kwargs["source_revision"], + sent_usage_generations=sorted(kwargs["applied_usage"]), + last_flush_at=0) + + try: + cursor_flush.STATE_DIR = root / "state" + cursor_flush._CURSOR_PROJECTS = projects + cursor_flush.cursor_reader._CHATS = root / "chats" + cursor_flush.cursor_reader._PROJECTS = projects + cursor_flush._flush = fake_flush + cursor_flush._log = lambda _message: None + for row in fixture["invocations"]: + rows = fixture["transcripts"][row["transcript"]] + path.write_text("".join(json.dumps(r) + "\n" for r in rows), + encoding="utf-8") + payload = dict(row["payload"], transcript_path=str(path)) + assert _run_main(row["event"], payload) == 0 + state = cursor_flush._read_state(session, strict=True) + finally: + (cursor_flush.STATE_DIR, cursor_flush._CURSOR_PROJECTS, + cursor_flush.cursor_reader._CHATS, + cursor_flush.cursor_reader._PROJECTS, + cursor_flush._flush, cursor_flush._log) = originals + + stored = {generation: (event["usage"]["input_tokens"], + event["usage"]["cache_read_input_tokens"], + event["basis"]) + for generation, event in state["usage_events"].items()} + # Gross IDE input minus its cache reads: 18356-6272, 102136-72320, + # 145087-143616. Turn 3 was a one-line edit on a cached context. + assert stored == { + "c3e427e5-7e9a-45db-8b49-56016bbc68d0": (12084, 6272, "net"), + "e46407dd-f24c-4480-9690-901e3193d891": (29816, 72320, "net"), + "f0b04523-791b-45a4-9af8-7d9328b2af86": (1471, 143616, "net"), + }, stored + final = [r for r in sent[-1] if r.get("type") == "assistant"][-1] + assert final["message"]["usage"]["input_tokens"] == 1471 + print("PASS test_recorded_ide_session_stores_net_usage_per_turn") + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): diff --git a/tests/fixtures/cursor_ide_subagent_taps.jsonl b/tests/fixtures/cursor_ide_subagent_taps.jsonl new file mode 100644 index 00000000..afe7a190 --- /dev/null +++ b/tests/fixtures/cursor_ide_subagent_taps.jsonl @@ -0,0 +1,15 @@ +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "beforeSubmitPrompt", "model": "grok-4.7-high-fast", "model_id": "grok-4.7", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "preToolUse", "model": "", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "tool_name": "Task", "tool_use_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "hook_event_name": "subagentStart", "is_parallel_worker": false, "model": "composer-2.5-fast", "parent_conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "subagent_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "subagent_model": "composer-2.5-fast", "subagent_type": "explore", "tool_call_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "preToolUse", "model": "", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "tool_name": "Task", "tool_use_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "hook_event_name": "subagentStart", "is_parallel_worker": false, "model": "composer-2.5-fast", "parent_conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "subagent_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "subagent_model": "composer-2.5-fast", "subagent_type": "explore", "tool_call_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "5303a535-740e-454f-935c-4fdcaa086faf", "cursor_version": "3.25.6", "generation_id": "1b7cb00b-a928-41eb-9cc9-0b7fe8ec981c", "hook_event_name": "preToolUse", "model": "composer-2.5-fast", "parent_tool_call_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "session_id": "5303a535-740e-454f-935c-4fdcaa086faf", "tool_name": "Read", "tool_use_id": "tool_e0225ba8-fa1a-4e2e-917f-490bfd9474f", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "38755d32-098d-4cb4-991e-a7c15bd93594", "cursor_version": "3.25.6", "generation_id": "0288a62e-198e-4247-b8d7-da9b49ca8809", "hook_event_name": "preToolUse", "model": "composer-2.5-fast", "parent_tool_call_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "session_id": "38755d32-098d-4cb4-991e-a7c15bd93594", "tool_name": "Read", "tool_use_id": "tool_eb8d325f-5285-4417-84ff-520bad14e7d", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "5303a535-740e-454f-935c-4fdcaa086faf", "cursor_version": "3.25.6", "generation_id": "1b7cb00b-a928-41eb-9cc9-0b7fe8ec981c", "hook_event_name": "postToolUse", "model": "composer-2.5-fast", "parent_tool_call_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "session_id": "5303a535-740e-454f-935c-4fdcaa086faf", "tool_name": "Read", "tool_use_id": "tool_e0225ba8-fa1a-4e2e-917f-490bfd9474f", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "38755d32-098d-4cb4-991e-a7c15bd93594", "cursor_version": "3.25.6", "generation_id": "0288a62e-198e-4247-b8d7-da9b49ca8809", "hook_event_name": "postToolUse", "model": "composer-2.5-fast", "parent_tool_call_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "session_id": "38755d32-098d-4cb4-991e-a7c15bd93594", "tool_name": "Read", "tool_use_id": "tool_eb8d325f-5285-4417-84ff-520bad14e7d", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "postToolUse", "model": "composer-2.5-fast", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "tool_name": "Task", "tool_use_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "postToolUse", "model": "composer-2.5-fast", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "tool_name": "Task", "tool_use_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"agent_transcript_path": null, "child_conversation_id": "5303a535-740e-454f-935c-4fdcaa086faf", "conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "description": "Read calc.py function", "duration_ms": 4597, "generation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "hook_event_name": "subagentStop", "loop_count": 0, "model": "composer-2.5-fast", "parent_conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "status": "completed", "subagent_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-1\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_0", "subagent_type": "explore", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"agent_transcript_path": null, "child_conversation_id": "38755d32-098d-4cb4-991e-a7c15bd93594", "conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "description": "Read mathy.py function", "duration_ms": 3915, "generation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "hook_event_name": "subagentStop", "loop_count": 0, "model": "composer-2.5-fast", "parent_conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "status": "completed", "subagent_id": "call-15eccc92-43df-4677-b80a-66cb399f1599-2\nfc_a7283783-2f98-98d9-b46c-b703af70b28f_1", "subagent_type": "explore", "transcript_path": null, "workspace_roots": ["{WORKSPACE}"]} +{"cache_read_tokens": 46208, "cache_write_tokens": 0, "conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "afterAgentResponse", "input_tokens": 62860, "model": "grok-4.7-high-fast", "model_id": "grok-4.7", "output_tokens": 1535, "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "transcript_path": "{PROJECTS}/ws/agent-transcripts/14dc53eb-2630-4480-b660-c3d3e28ec926/14dc53eb-2630-4480-b660-c3d3e28ec926.jsonl", "workspace_roots": ["{WORKSPACE}"]} +{"cache_read_tokens": 46208, "cache_write_tokens": 0, "conversation_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "cursor_version": "3.25.6", "generation_id": "4eb1022f-772f-455c-b171-8d49a7c5d78b", "hook_event_name": "stop", "input_tokens": 62860, "loop_count": 0, "model": "grok-4.7-high-fast", "model_id": "grok-4.7", "output_tokens": 1535, "session_id": "14dc53eb-2630-4480-b660-c3d3e28ec926", "status": "completed", "transcript_path": "{PROJECTS}/ws/agent-transcripts/14dc53eb-2630-4480-b660-c3d3e28ec926/14dc53eb-2630-4480-b660-c3d3e28ec926.jsonl", "workspace_roots": ["{WORKSPACE}"]} diff --git a/tests/fixtures/cursor_ide_usage_recording.json b/tests/fixtures/cursor_ide_usage_recording.json new file mode 100644 index 00000000..586a5a01 --- /dev/null +++ b/tests/fixtures/cursor_ide_usage_recording.json @@ -0,0 +1,611 @@ +{ + "about": "Cursor IDE 3.25.6 (grok-4.7-high-fast), three turns recorded with harness/drivers/cursor_record.py on 2026-10-17. Only the usage-bearing afterAgentResponse/stop invocations are kept, each with the transcript as it stood when the hook fired; paths and the account email are scrubbed. input_tokens on these hooks is GROSS (it includes cache_read_tokens).", + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "transcripts": { + "4d3909476bc2157a3ed506c3156fe1215bf72d61b2169e68e0f138e07136049b": [ + { + "role": "user", + "message": { + "content": [ + { + "type": "text", + "text": "Friday, Oct 16, 2026, 7:40 PM (UTC-7)\n\nReply with the single word ONE.\n" + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "ONE" + } + ] + } + }, + { + "type": "turn_ended", + "status": "success" + } + ], + "7349c3c5d444d212a52b8dd74b5ecd088b574a745127cb54e83fbb5d2a77be6f": [ + { + "role": "user", + "message": { + "content": [ + { + "type": "text", + "text": "Friday, Oct 16, 2026, 7:40 PM (UTC-7)\n\nReply with the single word ONE.\n" + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "ONE" + } + ] + } + }, + { + "type": "turn_ended", + "status": "success" + }, + { + "role": "user", + "message": { + "content": [ + { + "type": "text", + "text": "Friday, Oct 16, 2026, 7:41 PM (UTC-7)\n\nRead every .py file under /src/agent-plugins-internal/plugins/memhub-staging/scripts/readers and summarize each in one line. This makes the context big enough that Cursor reuses cached input.\n" + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "I'll list every Python file in that readers directory and summarize each one in a single line." + }, + { + "type": "tool_use", + "name": "Glob", + "input": { + "target_directory": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers", + "glob_pattern": "**/*.py" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/strict_json.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/jsonl.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/discovery.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/cursor.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex_usage.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex_history.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/claude.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/__init__.py" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "Two files were too large to read in full. I’ll take their structure from the first section and public definitions so each still gets a one-line summary." + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/cursor.py", + "limit": 120 + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex.py", + "limit": 120 + } + }, + { + "type": "tool_use", + "name": "Grep", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/cursor.py", + "pattern": "^(def |class |HOST |\"\"\")" + } + }, + { + "type": "tool_use", + "name": "Grep", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex.py", + "pattern": "^(def |class |HOST |\"\"\")" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "⛔️ Rule fired: Delegate big file reads to a cheaper model\n⛔️ Rule fired: Delegate big file reads to a cheaper model\n\n- `__init__.py` — Registry of host readers (Claude, Codex, Cursor) plus host sniffing and the shared Claude-shaped record contract check.\n- `claude.py` — Reads `~/.claude/projects` JSONL transcripts as the canonical shape, listing, locating, and extracting session cwd without transforming records.\n- `codex.py` — Reads Codex rollout JSONL and reshapes Responses-API items (text, reasoning, tool calls, token usage) into Claude records.\n- `codex_history.py` — Stitches paginated Codex rollouts that share a history chain, projecting inherited context and carrying usage baselines across segments.\n- `codex_usage.py` — Overlays persisted per-response token ledgers onto Codex records, replacing parallel UI token-meter snapshots from the first ledger onward.\n- `cursor.py` — Reads Cursor `store.db` hash trees and hook transcripts into Claude records, stripping injected context and keeping only real timestamps.\n- `discovery.py` — Walks a selected filesystem layout for discovery, reporting inaccessible paths, symlinks, and non-regular files instead of following them.\n- `jsonl.py` — Opens JSONL with universal newlines and surrogate-escaped bytes, then enforces a per-record byte bound.\n- `strict_json.py` — Optional strict `json.loads` that rejects non-finite numbers and non-standard constants while leaving tolerant capture unchanged." + } + ] + } + }, + { + "type": "turn_ended", + "status": "success" + } + ], + "acadfa60a9f2ed62d5bf0d34798d760835c1ec94fee4bfa6772552958b16117b": [ + { + "role": "user", + "message": { + "content": [ + { + "type": "text", + "text": "Friday, Oct 16, 2026, 7:40 PM (UTC-7)\n\nReply with the single word ONE.\n" + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "ONE" + } + ] + } + }, + { + "type": "turn_ended", + "status": "success" + }, + { + "role": "user", + "message": { + "content": [ + { + "type": "text", + "text": "Friday, Oct 16, 2026, 7:41 PM (UTC-7)\n\nRead every .py file under /src/agent-plugins-internal/plugins/memhub-staging/scripts/readers and summarize each in one line. This makes the context big enough that Cursor reuses cached input.\n" + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "I'll list every Python file in that readers directory and summarize each one in a single line." + }, + { + "type": "tool_use", + "name": "Glob", + "input": { + "target_directory": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers", + "glob_pattern": "**/*.py" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/strict_json.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/jsonl.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/discovery.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/cursor.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex_usage.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex_history.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/claude.py" + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/__init__.py" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "Two files were too large to read in full. I’ll take their structure from the first section and public definitions so each still gets a one-line summary." + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/cursor.py", + "limit": 120 + } + }, + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex.py", + "limit": 120 + } + }, + { + "type": "tool_use", + "name": "Grep", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/cursor.py", + "pattern": "^(def |class |HOST |\"\"\")" + } + }, + { + "type": "tool_use", + "name": "Grep", + "input": { + "path": "/src/agent-plugins-internal/plugins/memhub-staging/scripts/readers/codex.py", + "pattern": "^(def |class |HOST |\"\"\")" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "⛔️ Rule fired: Delegate big file reads to a cheaper model\n⛔️ Rule fired: Delegate big file reads to a cheaper model\n\n- `__init__.py` — Registry of host readers (Claude, Codex, Cursor) plus host sniffing and the shared Claude-shaped record contract check.\n- `claude.py` — Reads `~/.claude/projects` JSONL transcripts as the canonical shape, listing, locating, and extracting session cwd without transforming records.\n- `codex.py` — Reads Codex rollout JSONL and reshapes Responses-API items (text, reasoning, tool calls, token usage) into Claude records.\n- `codex_history.py` — Stitches paginated Codex rollouts that share a history chain, projecting inherited context and carrying usage baselines across segments.\n- `codex_usage.py` — Overlays persisted per-response token ledgers onto Codex records, replacing parallel UI token-meter snapshots from the first ledger onward.\n- `cursor.py` — Reads Cursor `store.db` hash trees and hook transcripts into Claude records, stripping injected context and keeping only real timestamps.\n- `discovery.py` — Walks a selected filesystem layout for discovery, reporting inaccessible paths, symlinks, and non-regular files instead of following them.\n- `jsonl.py` — Opens JSONL with universal newlines and surrogate-escaped bytes, then enforces a per-record byte bound.\n- `strict_json.py` — Optional strict `json.loads` that rejects non-finite numbers and non-standard constants while leaving tolerant capture unchanged." + } + ] + } + }, + { + "type": "turn_ended", + "status": "success" + }, + { + "role": "user", + "message": { + "content": [ + { + "type": "text", + "text": "Friday, Oct 16, 2026, 7:42 PM (UTC-7)\n\nAppend a line # reviewed to fixture.py, then reply DONE\n" + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "I'll add `# reviewed` to `fixture.py` and then reply DONE." + }, + { + "type": "tool_use", + "name": "Glob", + "input": { + "glob_pattern": "**/fixture.py" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "Read", + "input": { + "path": "/tmp/memhub-cursor-recording/fixture.py" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "name": "StrReplace", + "input": { + "path": "/tmp/memhub-cursor-recording/fixture.py", + "old_string": "CANARY = \"memhub-cursor-recording\"", + "new_string": "CANARY = \"memhub-cursor-recording\"\n# reviewed" + } + } + ] + } + }, + { + "role": "assistant", + "message": { + "content": [ + { + "type": "text", + "text": "DONE" + } + ] + } + }, + { + "type": "turn_ended", + "status": "success" + } + ] + }, + "invocations": [ + { + "seq": 5, + "event": "afterAgentResponse", + "transcript": "4d3909476bc2157a3ed506c3156fe1215bf72d61b2169e68e0f138e07136049b", + "payload": { + "conversation_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "generation_id": "c3e427e5-7e9a-45db-8b49-56016bbc68d0", + "model": "grok-4.7-high-fast", + "input_tokens": 18356, + "output_tokens": 27, + "cache_read_tokens": 6272, + "cache_write_tokens": 0, + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "hook_event_name": "afterAgentResponse", + "cursor_version": "3.25.6", + "workspace_roots": [ + "/tmp/memhub-cursor-recording" + ], + "text": "ONE" + } + }, + { + "seq": 6, + "event": "stop", + "transcript": "4d3909476bc2157a3ed506c3156fe1215bf72d61b2169e68e0f138e07136049b", + "payload": { + "conversation_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "generation_id": "c3e427e5-7e9a-45db-8b49-56016bbc68d0", + "model": "grok-4.7-high-fast", + "input_tokens": 18356, + "output_tokens": 27, + "cache_read_tokens": 6272, + "cache_write_tokens": 0, + "status": "completed", + "loop_count": 0, + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "hook_event_name": "stop", + "cursor_version": "3.25.6", + "workspace_roots": [ + "/tmp/memhub-cursor-recording" + ] + } + }, + { + "seq": 55, + "event": "afterAgentResponse", + "transcript": "7349c3c5d444d212a52b8dd74b5ecd088b574a745127cb54e83fbb5d2a77be6f", + "payload": { + "conversation_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "generation_id": "e46407dd-f24c-4480-9690-901e3193d891", + "model": "grok-4.7-high-fast", + "input_tokens": 102136, + "output_tokens": 2304, + "cache_read_tokens": 72320, + "cache_write_tokens": 0, + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "hook_event_name": "afterAgentResponse", + "cursor_version": "3.25.6", + "workspace_roots": [ + "/tmp/memhub-cursor-recording" + ], + "text": "⛔️ Rule fired: Delegate big file reads to a cheaper model\n⛔️ Rule fired: Delegate big file reads to a cheaper model\n\n- `__init__.py` — Registry of host readers (Claude, Codex, Cursor) plus host sniffing and the shared Claude-shaped record contract check.\n- `claude.py` — Reads `~/.claude/projects` JSONL transcripts as the canonical shape, listing, locating, and extracting session cwd without transforming records.\n- `codex.py` — Reads Codex rollout JSONL and reshapes Responses-API items (text, reasoning, tool calls, token usage) into Claude records.\n- `codex_history.py` — Stitches paginated Codex rollouts that share a history chain, projecting inherited context and carrying usage baselines across segments.\n- `codex_usage.py` — Overlays persisted per-response token ledgers onto Codex records, replacing parallel UI token-meter snapshots from the first ledger onward.\n- `cursor.py` — Reads Cursor `store.db` hash trees and hook transcripts into Claude records, stripping injected context and keeping only real timestamps.\n- `discovery.py` — Walks a selected filesystem layout for discovery, reporting inaccessible paths, symlinks, and non-regular files instead of following them.\n- `jsonl.py` — Opens JSONL with universal newlines and surrogate-escaped bytes, then enforces a per-record byte bound.\n- `strict_json.py` — Optional strict `json.loads` that rejects non-finite numbers and non-standard constants while leaving tolerant capture unchanged." + } + }, + { + "seq": 56, + "event": "stop", + "transcript": "7349c3c5d444d212a52b8dd74b5ecd088b574a745127cb54e83fbb5d2a77be6f", + "payload": { + "conversation_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "generation_id": "e46407dd-f24c-4480-9690-901e3193d891", + "model": "grok-4.7-high-fast", + "input_tokens": 102136, + "output_tokens": 2304, + "cache_read_tokens": 72320, + "cache_write_tokens": 0, + "status": "completed", + "loop_count": 0, + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "hook_event_name": "stop", + "cursor_version": "3.25.6", + "workspace_roots": [ + "/tmp/memhub-cursor-recording" + ] + } + }, + { + "seq": 77, + "event": "afterAgentResponse", + "transcript": "acadfa60a9f2ed62d5bf0d34798d760835c1ec94fee4bfa6772552958b16117b", + "payload": { + "conversation_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "generation_id": "f0b04523-791b-45a4-9af8-7d9328b2af86", + "model": "grok-4.7-high-fast", + "input_tokens": 145087, + "output_tokens": 295, + "cache_read_tokens": 143616, + "cache_write_tokens": 0, + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "hook_event_name": "afterAgentResponse", + "cursor_version": "3.25.6", + "workspace_roots": [ + "/tmp/memhub-cursor-recording" + ], + "text": "DONE" + } + }, + { + "seq": 78, + "event": "stop", + "transcript": "acadfa60a9f2ed62d5bf0d34798d760835c1ec94fee4bfa6772552958b16117b", + "payload": { + "conversation_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "generation_id": "f0b04523-791b-45a4-9af8-7d9328b2af86", + "model": "grok-4.7-high-fast", + "input_tokens": 145087, + "output_tokens": 295, + "cache_read_tokens": 143616, + "cache_write_tokens": 0, + "status": "completed", + "loop_count": 0, + "session_id": "369081f8-d23c-4081-8f17-6e109ba6346f", + "hook_event_name": "stop", + "cursor_version": "3.25.6", + "workspace_roots": [ + "/tmp/memhub-cursor-recording" + ] + } + } + ] +} diff --git a/tests/fixtures/rewind_native_probe.py b/tests/fixtures/rewind_native_probe.py new file mode 100644 index 00000000..fcf443b5 --- /dev/null +++ b/tests/fixtures/rewind_native_probe.py @@ -0,0 +1,334 @@ +"""Real Codex binary + localhost fixture provider. No real model inference.""" +import argparse, json, os, sys, threading, time, uuid, subprocess, shlex +from pathlib import Path +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +parser=argparse.ArgumentParser() +parser.add_argument('--codex',required=True) +parser.add_argument('--plugin-repo',default=str(Path(__file__).resolve().parents[2])) +parser.add_argument('--out',required=True) +parser.add_argument('--workspace-write',action='store_true') +parser.add_argument('--extra-writable-root',action='store_true') +parser.add_argument('--official-process-wrappers',action='store_true',help='Exercise the shipped plugin isolation implementation') +parser.add_argument('--isolated-rewind-home',action='store_true') +parser.add_argument('--plugin-integrations',action='store_true') +parser.add_argument('--probe-hook-isolation',action='store_true', + help='Require a retained hook and its subprocess to be unable to write the original checkout') +args=parser.parse_args() +if args.probe_hook_isolation: + args.official_process_wrappers=True;args.isolated_rewind_home=True +if args.plugin_integrations and not args.probe_hook_isolation: + parser.error('--plugin-integrations requires --probe-hook-isolation') +sys.path.insert(0,str(Path(args.plugin_repo).resolve())) +from harness.drivers.codex_appserver import AppServer +from harness.drivers.codex import review_hooks +probe_root=Path(args.out).resolve();probe_root.mkdir(parents=True,exist_ok=False) +source=probe_root/'source';branch=probe_root/'branch';source.mkdir(exist_ok=True);branch.mkdir(exist_ok=True) +extra=probe_root/'artifacts';extra.mkdir() +if args.extra_writable_root and not args.workspace_write: + parser.error('--extra-writable-root requires --workspace-write') +(source/'tracked.txt').write_text('original fixture\n');(branch/'tracked.txt').write_text('original fixture\n') +subprocess.run(['git','init','-q'],cwd=source,check=True) +subprocess.run(['git','-c','user.name=Fixture','-c','user.email=fixture@invalid','add','.'],cwd=source,check=True) +subprocess.run(['git','-c','user.name=Fixture','-c','user.email=fixture@invalid','commit','-qm','fixture'],cwd=source,check=True) +sys.path.insert(0,str(Path(args.plugin_repo)/'plugins/memhub/scripts')) +from rewind.codex import Codex +from rewind.checkpoints import Store +requests=[] +write_probe_sent=False +hook_log=probe_root/'hook-events.jsonl' +hook_log.write_text('') +hook_script=probe_root/'checkpoint_hook.py' +hook_script.write_text("import sys,json\nfrom pathlib import Path\np=json.load(sys.stdin)\nwith Path("+repr(str(hook_log))+").open('a') as f: f.write(json.dumps(p)+'\\n')\n") +hook_script.write_text(hook_script.read_text()+"\nsys.path.insert(0,"+repr(str(Path(args.plugin_repo)/'plugins/memhub/scripts'))+")\nfrom rewind.hooks import capture_prompt\nif p.get('hook_event_name') == 'UserPromptSubmit': capture_prompt(p)\n") +if args.probe_hook_isolation: + hook_script.write_text(hook_script.read_text()+f''' +if Path(p['cwd']).resolve() == Path({str(branch)!r}): + import subprocess + attempts = [] + targets = [{str(source/'forbidden-hook-write')!r}, {str(source/'.git'/'forbidden-hook-write')!r}] + transcript_file = Path({str(probe_root/'protected-transcript-path')!r}) + if transcript_file.exists(): targets.append(transcript_file.read_text()) + for target in targets: + code = 'from pathlib import Path; Path(' + repr(target) + ').write_text("hook escaped")' + child = subprocess.run([sys.executable, '-c', code], capture_output=True) + attempts.append({{'target': target, 'denied': child.returncode != 0 and b'PermissionError' in child.stderr}}) + with Path({str(probe_root/'hook-isolation.jsonl')!r}).open('a') as f: + f.write(json.dumps({{'event': p['hook_event_name'], 'session_id':p.get('session_id'), 'plugin_fixture':p.get('plugin_fixture',False), 'attempts': attempts}}) + '\\n') +''') +if args.probe_hook_isolation: + process_probe=probe_root/'process_probe.py' + process_probe.write_text(f'''import json, os, subprocess, sys +from pathlib import Path +with Path({str(probe_root/'process-launches.jsonl')!r}).open('a') as f: f.write(json.dumps(dict(mode=sys.argv[1],cwd=os.getcwd(),home=os.environ.get('CODEX_HOME')))+'\\n') + +def probe(event, cwd): + if Path(cwd).resolve() != Path({str(branch)!r}) and not (event == 'plugin_stdio' and Path(os.environ.get('CODEX_HOME','')).is_relative_to(Path({str(probe_root/'plugin-state/rewind/clients')!r}))): return + attempts=[] + targets = [{str(source/'forbidden-process-write')!r}, {str(source/'.git'/'forbidden-process-write')!r}] + transcript_file = Path({str(probe_root/'protected-transcript-path')!r}) + if transcript_file.exists(): targets.append(transcript_file.read_text()) + for target in targets: + code='from pathlib import Path; Path('+repr(target)+').write_text("escaped")' + child=subprocess.run([sys.executable,'-c',code],capture_output=True) + attempts.append({{"target":target,"denied":child.returncode != 0 and b"PermissionError" in child.stderr}}) + with Path({str(probe_root/'process-isolation.jsonl')!r}).open('a') as log: + log.write(json.dumps({{"event":event,"attempts":attempts}})+"\\n") + +if sys.argv[1]=='notify': + probe('notify',json.loads(sys.argv[-1])['cwd']) +else: + for line in sys.stdin: + request=json.loads(line) + if 'id' not in request: continue + method=request['method'] + if method=='initialize': + probe(sys.argv[1],os.getcwd()) + result={{"protocolVersion":request['params']['protocolVersion'],"capabilities":{{"tools":{{}}}},"serverInfo":{{"name":"source-guard-fixture","version":"1"}}}} + elif method=='tools/list': result={{"tools":[]}} + else: result={{}} + print(json.dumps({{"jsonrpc":"2.0","id":request['id'],"result":result}}),flush=True) +''') +class Handler(BaseHTTPRequestHandler): + def log_message(self,*args): pass + def do_POST(self): + global write_probe_sent + body=json.loads(self.rfile.read(int(self.headers['Content-Length'])));requests.append({**body,'_hook_count':sum(json.loads(line).get('hook_event_name')=='UserPromptSubmit' for line in hook_log.read_text().splitlines())}) + rid='resp_'+uuid.uuid4().hex;mid='msg_'+uuid.uuid4().hex + item={'id':mid,'type':'message','role':'assistant','status':'completed','content':[{'type':'output_text','text':'fixture answer','annotations':[]}]} + response={'id':rid,'object':'response','created_at':int(time.time()),'status':'completed','model':body.get('model'),'output':[item],'usage':{'input_tokens':10,'output_tokens':3,'total_tokens':13}} + events=[('response.created',{'response':{**response,'status':'in_progress','output':[]}}),('response.output_item.added',{'output_index':0,'item':{**item,'status':'in_progress','content':[]}}),('response.output_text.delta',{'item_id':mid,'output_index':0,'content_index':0,'delta':'fixture answer'}),('response.output_item.done',{'output_index':0,'item':item}),('response.completed',{'response':response})] + if args.workspace_write and body.get('model')=='fixture-b' and not write_probe_sent: + write_probe_sent=True + available=[t.get('name') for t in body.get('tools',[]) if t.get('type')=='function'] + assert 'exec_command' in available, available + command='printf branch-write > '+shlex.quote(str(branch/'write-proof'))+'; if printf forbidden > '+shlex.quote(str(source/'forbidden-write'))+' 2>/dev/null; then exit 91; fi' + if args.extra_writable_root: + command += '; printf extra-write > '+shlex.quote(str(extra/'write-proof')) + arguments=json.dumps({'cmd':command,'max_output_tokens':1000}) + item={'type':'function_call','id':'fc_'+uuid.uuid4().hex,'call_id':'call_'+uuid.uuid4().hex,'name':'exec_command','arguments':arguments,'status':'completed'} + response['output']=[item] + events=[('response.created',{'response':{**response,'status':'in_progress','output':[]}}), + ('response.output_item.added',{'output_index':0,'item':{**item,'status':'in_progress','arguments':''}}), + ('response.function_call_arguments.delta',{'item_id':item['id'],'output_index':0,'delta':arguments}), + ('response.output_item.done',{'output_index':0,'item':item}),('response.completed',{'response':response})] + raw=''.join('event: '+typ+'\ndata: '+json.dumps({'type':typ,**data})+'\n\n' for typ,data in events).encode() + self.send_response(200);self.send_header('Content-Type','text/event-stream');self.send_header('Content-Length',str(len(raw)));self.end_headers();self.wfile.write(raw) +server=ThreadingHTTPServer(('127.0.0.1',0),Handler);threading.Thread(target=server.serve_forever,daemon=True).start() +client_home=probe_root/'client';client_home.mkdir(exist_ok=True) +(client_home/'config.toml').write_text(f'''model_provider = "rewind_fixture" +model = "fixture-a" +[model_providers.rewind_fixture] +name = "Rewind local fixture" +base_url = "http://127.0.0.1:{server.server_port}/v1" +wire_api = "responses" +requires_openai_auth = false +supports_websockets = false +''') +if args.probe_hook_isolation: + config_text=(client_home/'config.toml').read_text().replace('model = "fixture-a"\n', + 'model = "fixture-a"\nnotify = '+json.dumps([sys.executable,str(process_probe),'notify'])+'\n') + config_text+='\n[mcp_servers.source_guard_fixture]\ncommand = '+json.dumps(sys.executable)+'\nargs = '+json.dumps([str(process_probe),'stdio'])+'\n' + (client_home/'config.toml').write_text(config_text) +handlers={'UserPromptSubmit':[{'hooks':[{'type':'command','command':sys.executable+' '+str(hook_script),'timeout':10}]}]} +if args.probe_hook_isolation: + handlers['SessionStart']=[{'hooks':[{'type':'command','command':sys.executable+' '+str(hook_script),'timeout':10}]}] + handlers['PostToolUse']=[{'hooks':[{'type':'command','command':sys.executable+' '+str(hook_script),'timeout':10,'async':True}]}] +(client_home/'hooks.json').write_text(json.dumps({'hooks':handlers})) +if args.plugin_integrations: + plugin=client_home/'plugins/cache/fixture/probe/1.0.0' + (plugin/'.codex-plugin').mkdir(parents=True) + (plugin/'.codex-plugin/plugin.json').write_text(json.dumps({'name':'probe','version':'1.0.0','hooks':'./hooks/hooks.json'})) + (plugin/'hooks').mkdir() + (plugin/'hook.py').write_text(hook_script.read_text().replace('p=json.load(sys.stdin)',"p=json.load(sys.stdin);p['plugin_fixture']=True")) + (plugin/'hooks/hooks.json').write_text(json.dumps({'hooks':{'SessionStart':[{'hooks':[{'type':'command','command':sys.executable+' "${PLUGIN_ROOT}/hook.py"','timeout':10}]}]}})) + (plugin/'process.py').write_text(process_probe.read_text()) + (plugin/'.mcp.json').write_text(json.dumps({'mcpServers':{'plugin_fixture':{'command':sys.executable,'args':[str(plugin/'process.py'),'plugin_stdio']}}})) + with (client_home/'config.toml').open('a') as f:f.write('\n[features]\nplugins = true\n[plugins."probe@fixture"]\nenabled = true\n') +os.environ['MEMHUB_CONFIG_DIR']=str(probe_root/'plugin-state') +from rewind.state import root,write,read,digest +write(root()/'settings.json',{'enabled':True}) +env={**os.environ,'CODEX_HOME':str(client_home)} +rpc=AppServer([args.codex,'app-server'],cwd=source,env=env,log=probe_root/'stderr.log') +summary={'binary':args.codex,'provider':'local deterministic fixture; no real model inference'} +def turn(tid,text,image=False): + result=rpc.request('turn/start',{'threadId':tid,'clientUserMessageId':str(uuid.uuid4()),'input':[{'type':'text','text':text}]+([{'type':'image','url':'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aPioAAAAASUVORK5CYII='}] if image else [])},timeout_s=20) + deadline=time.time()+25 + while time.time()deadline: raise AssertionError('Refork hook never ran') + time.sleep(.05) + attempts=[item for row in rows for item in row['attempts']] + summary['hook_source_writes_denied']=bool(attempts) and all(item['denied'] for item in attempts) + assert summary['hook_source_writes_denied'], 'Native hook subprocess wrote the original checkout or Git directory' + assert {'SessionStart','UserPromptSubmit'} <= {row['event'] for row in rows[pre_restart_rows:]}, 'Hooks did not run after process restart' + if args.workspace_write: + assert any(row['event']=='PostToolUse' for row in rows), 'Async tool hook did not execute' + summary['hook_protection_survived_process_restart']=True + summary['hook_protection_survived_refork']=True + if args.plugin_integrations: + assert any(row.get('plugin_fixture') for row in rows), 'Plugin hook did not execute' + summary['plugin_hook_protected']=True + deadline=time.time()+10 + while True: + process_rows=[json.loads(line) for line in (probe_root/'process-isolation.jsonl').read_text().splitlines()] if (probe_root/'process-isolation.jsonl').exists() else [] + required={'stdio','notify'} | ({'plugin_stdio'} if args.plugin_integrations else set()) + if required <= {row['event'] for row in process_rows}: break + if time.time()>deadline: raise AssertionError('Protected stdio and notification processes did not both run') + time.sleep(.05) + assert all(item['denied'] for row in process_rows for item in row['attempts']), 'Local subprocess wrote source or Git storage' + summary['stdio_and_notify_source_writes_denied']=True + summary['plugin_stdio_protected']=args.plugin_integrations + assert resumed['thread']['id']==child + if args.isolated_rewind_home: + assert (original_client_home/'config.toml').read_bytes()==original_config + assert (original_client_home/'hooks.json').read_bytes()==original_hooks + assert original_transcript.read_bytes()==original_transcript_bytes + summary['separate_rewind_home_original_config_and_transcript_unchanged']=True + else: + after=rpc.request('thread/read',{'threadId':tid,'includeTurns':True}) + assert before['thread']['turns']==after['thread']['turns'] + assert (source/'tracked.txt').read_text()=='later source state\n' + assert any(r.get('model')=='fixture-b' for r in requests) + from rewind.capture import prepare + from rewind.runner import register_branch + from rewind.state import branch_path + from rewind.hooks import metadata + source_path=Path(checkpoint['local_config']['transcript_path']) + source_records,_,_=prepare(source_path,tid) + selected=[r for r in source_records if r.get('rewind_v1',{}).get('checkpoint',{}).get('native_turn_id')==t2] + assert len(selected)==1, 'exact native turn checkpoint absent from canonical capture' + selected_event=selected[0]['uuid'] + journal={'comparison_id':str(uuid.uuid4()),'run_id':str(uuid.uuid4()),'claim_id':str(uuid.uuid4()), + 'plan':{'source':{'workspace_id':str(uuid.uuid4()),'conv_id':str(uuid.uuid4())}, + 'boundary':{'selected_event':{'event_id':selected_event,'capture_version':1}}}} + register_branch(journal,checkpoint,native) + child_path=next(client_home.joinpath('sessions').rglob('*'+child+'.jsonl')) + child_records,_,lineage=prepare(child_path,child) + assert lineage['native_branch_id']==child and lineage['prefix_mapping']==[] + assert lineage['continuation_start_event_id']==child_records[0]['uuid'] + assert all(r['native_event_id']==r['uuid'] for r in child_records) + summary['capture_checkpoint_and_lineage']=True + if args.official_process_wrappers: + assert hashlib.sha256(Path(args.codex).read_bytes()).hexdigest()==summary['official_binary_sha256'] + summary['official_binary_unchanged']=True + summary.update({'pass':True,'original_turns':3,'fork_prefix_turns':len(fork_turns),'continued_turns':3 if args.probe_hook_isolation else 2,'source_preserved':True,'model_requests':[r.get('model') for r in requests],'source_id':tid,'branch_id':child,'fork_response_model':fork.get('model'),'checkpoint_before_selected_turn':True,'workspace_write':args.workspace_write,'branch_cwd':resumed.get('cwd'),'source_root_id':start['thread'].get('sessionId'),'fork_root_id':fork['thread'].get('sessionId')}) +except Exception as e: + summary.update({'pass':False,'error':str(e),'requests_seen':len(requests)});raise +finally: + rpc.close();server.shutdown();(probe_root/'summary.json').write_text(json.dumps(summary,indent=2));print(json.dumps(summary)) diff --git a/tests/flush_session_test.py b/tests/flush_session_test.py index caf97b7e..9ff7d96a 100644 --- a/tests/flush_session_test.py +++ b/tests/flush_session_test.py @@ -1,8 +1,7 @@ #!/usr/bin/env python3 """What the whole-transcript flush puts on the wire, and when it gives up. -Run: uv run --with 'mcp<2' python plugins/memhub/scripts/flush_session_test.py -(the module imports the MCP SDK through ``_memhub_auth``) +Run: python3 tests/flush_session_test.py Covers the arguments assembly and the success/failure contract of one slice — the parts that decide whether a capture lands in the right brain, under the @@ -188,7 +187,7 @@ def run_main(raises, deadline="0.5"): real_flush, real_log = fs._flush, fs._log stdin = sys.stdin - async def fake_flush(session_id, transcript_path): + async def fake_flush(session_id, transcript_path, session_ended=False): if raises == "slow": await asyncio.sleep(10) # our wall clock wins raise TimeoutError("connection timed out") # an inner socket timeout @@ -253,7 +252,7 @@ def read(self): def _child_run(flag): calls = [] - async def fake_flush(session_id, transcript_path): + async def fake_flush(session_id, transcript_path, session_ended=False): calls.append(session_id) class FakeIn: @@ -277,6 +276,213 @@ def read(self): check("the same payload outside a child ships", _child_run("") == (0, ["fork-1"])) +# ── agents and workflow runs (MemHub-Backend agent-sessions spec §10.2) ── +# +# What a full flush sends for a session that spawned Agent-tool agents, MemHub +# harness forks and three dynamic workflows, against each server generation: +# one without agent capture, one with §4.1 only, and one with the details and +# runs. The server's tools are listed once, before the session goes. + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import agent_fixture as fx # noqa: E402 +import redact as _redact # noqa: E402 + +_AGENTS_41 = {"messages": {}, "agent_id": {}, "parent_conversation_id": {}} +SERVER_OLD = [{"name": "import_conversation", "inputSchema": {"properties": {"messages": {}}}}] +SERVER_41 = [{"name": "import_conversation", "inputSchema": {"properties": dict(_AGENTS_41)}}] +SERVER_FULL = [{"name": "import_conversation", "inputSchema": {"properties": { + **_AGENTS_41, "parent_agent_id": {}, "workflow_run_id": {}, "agent_status": {}, + "agent_result_preview": {}, "workflow_runs": {}}}}] +DETAIL_ARGS = {"parent_agent_id", "workflow_run_id", "agent_status", "agent_result_preview"} + + +class RecordingSession: + def __init__(self, *_a, **_k): + self.sent = RecordingSession.sent + + async def call_tool(self, name, arguments=None, timeout=None): + self.sent.append(dict(arguments)) + return FakeResult(structured={"conversation_id": arguments["conversation_id"], + "messages_received": len(arguments["messages"]), + "path": "agentic"}) + + +def flush_tree(tree, tools, state_dir, ended=True): + """One full flush of ``tree`` against a server listing ``tools``. + Returns (imports sent, list_tools call count).""" + RecordingSession.sent = [] + listed = [] + + def list_tools(url, bearer, *a, **k): + listed.append(1) + return tools + + real = (fs.resolve_bearer, fs.mcp_http.Session, fs.mcp_http.list_tools, + fs.make_slices, fs._SESSION_STATE_DIR, fs._log) + fs.resolve_bearer = lambda: ("https://example.invalid/mcp", "tok") + fs.mcp_http.Session = RecordingSession + fs.mcp_http.list_tools = list_tools + # Small slices, so the session goes in several and "the last" means something. + fs.make_slices = lambda records: real[3](records, chunk_bytes=4000) + fs._SESSION_STATE_DIR = state_dir + fs._log = lambda _m: None + try: + asyncio.run(fs._flush(fx.SESSION, str(tree.transcript), ended)) + finally: + (fs.resolve_bearer, fs.mcp_http.Session, fs.mcp_http.list_tools, + fs.make_slices, fs._SESSION_STATE_DIR, fs._log) = real + return RecordingSession.sent, len(listed) + + +def session_slices(sent): + return [a for a in sent if a["conversation_id"] == fx.SESSION] + + +def agent_imports(sent): + return [a for a in sent if a.get("parent_conversation_id") == fx.SESSION] + + +_root = Path(tempfile.mkdtemp(prefix="flush-agents-")) +_tree = fx.mirror_9b073366(_root / "proj") + +# (a) No agent capture on the server: no agents, no runs. +_state = _root / "state-a" +_state.mkdir() +sent, listed = flush_tree(_tree, SERVER_OLD, _state) +check("old server: tools listed once", listed == 1) +check("old server: no agents sent", agent_imports(sent) == []) +check("old server: no workflow_runs sent", not any("workflow_runs" in a for a in sent)) +check("old server: the session still goes", len(session_slices(sent)) > 1) + +# (b) §4.1 only: 74 agents with only the §4.1 arguments, no runs. +_state = _root / "state-b" +_state.mkdir() +sent, listed = flush_tree(_tree, SERVER_41, _state) +_agents = {a["conversation_id"] for a in agent_imports(sent)} +check("4.1 server: tools listed once", listed == 1) +check("4.1 server: 74 agents sent", len(_agents) == 74) +check("4.1 server: no harness fork sent", + not any(f in str(_agents) for f in fx.HARNESS_FORKS)) +check("4.1 server: no detail arguments", + not any(DETAIL_ARGS & set(a) for a in agent_imports(sent))) +check("4.1 server: no workflow_runs", not any("workflow_runs" in a for a in sent)) +_wf = [a for a in agent_imports(sent) if a.get("agent_type") == "workflow-subagent"] +check("4.1 server: workflow agents go as plain agents", + len({a["agent_id"] for a in _wf}) == 64 + and not any("parent_tool_use_id" in a for a in _wf)) +_cache = _json.loads((_state / f"{fx.SESSION}.agents").read_text()) +check("4.1 server: cache keys carry size, status and 'b'", + _cache[fx.TOOL_AGENTS[0]].endswith(":done:b") + and _cache["as00"].endswith(":-:b")) + +# The same server again: nothing re-sent. +sent, listed = flush_tree(_tree, SERVER_41, _state) +check("4.1 server: an unchanged agent is not re-sent", agent_imports(sent) == []) + +# (c) The server gains the details: every agent is re-sent once, with them. +sent, listed = flush_tree(_tree, SERVER_FULL, _state) +check("full server: tools listed once", listed == 1) +_imports = agent_imports(sent) +check("full server: each agent re-sent once on gaining details", + len({a["agent_id"] for a in _imports}) == 74) +_by = {} +for a in _imports: + _by.setdefault(a["agent_id"], a) +check("full server: tool agents carry status and result", + _by[fx.TOOL_AGENTS[0]].get("agent_status") == "done" + and len(_by[fx.TOOL_AGENTS[0]].get("agent_result_preview", "")) == 2000) +check("full server: workflow agents carry their run, never a status", + _by["as00"].get("workflow_run_id") == fx.RUN_SPLIT + and "agent_status" not in _by["as00"] + and _by["as00"].get("agent_description") == "design:A") +_slices = session_slices(sent) +check("full server: workflow_runs rides only on the last session slice", + len(_slices) > 1 and "workflow_runs" in _slices[-1] + and not any("workflow_runs" in a for a in _slices[:-1])) +check("full server: three runs sent", len(_slices[-1]["workflow_runs"]) == 3) +check("full server: never on an agent import", + not any("workflow_runs" in a for a in _imports)) +check("full server: runs are redacted", not _redact.contains_secret(_slices[-1]["workflow_runs"])) +sent, listed = flush_tree(_tree, SERVER_FULL, _state) +check("full server: runs re-sent on every flush, agents not", + "workflow_runs" in session_slices(sent)[-1] and agent_imports(sent) == [] and listed == 1) + +# A legacy integer cache entry never matches: that agent is re-sent once. +_cache = _json.loads((_state / f"{fx.SESSION}.agents").read_text()) +_cache[fx.TOOL_AGENTS[1]] = 12345 +(_state / f"{fx.SESSION}.agents").write_text(_json.dumps(_cache)) +sent, _ = flush_tree(_tree, SERVER_FULL, _state) +check("a legacy cache entry re-sends that agent once", + {a["agent_id"] for a in agent_imports(sent)} == {fx.TOOL_AGENTS[1]}) + +# A status change re-keys the cache: a running agent that finishes is re-sent. +_live = Path(tempfile.mkdtemp(prefix="flush-live-")) +_t2 = fx.Tree(_live / "proj") +_t2.records = [fx.assistant("r0", [fx.tool_use("toolu_L", "Agent", prompt="go")], "s0")] +_t2.write_session() +_mid = [fx.user("go", "u"), fx.assistant("r1", [fx.tool_use("t", "Bash", command="ls")], "a", + stop_reason="tool_use")] +_t2.agent("l1", _mid, agentType="general-purpose", toolUseId="toolu_L") +(_live / "state").mkdir() +sent, _ = flush_tree(_t2, SERVER_FULL, _live / "state", ended=False) +check("a running agent is sent as running", + [a.get("agent_status") for a in agent_imports(sent)] == ["running"]) +sent, _ = flush_tree(_t2, SERVER_FULL, _live / "state", ended=True) +check("the same bytes at session end are re-sent as stopped", + [a.get("agent_status") for a in agent_imports(sent)] == ["stopped"]) + +# No agents and no runs: the server's tools are not listed at all. +_bare = Path(tempfile.mkdtemp(prefix="flush-bare-")) +_t3 = fx.Tree(_bare / "proj") +_t3.records = [fx.user("hi", "p0")] +_t3.write_session() +(_bare / "state").mkdir() +sent, listed = flush_tree(_t3, SERVER_FULL, _bare / "state") +check("nothing to send beyond the session: no tools listed", listed == 0 and len(sent) == 1) + +# A fork's copy of its owner's spawning response never leaves the machine. +_fk = Path(tempfile.mkdtemp(prefix="flush-fork-")) +_t4 = fx.Tree(_fk / "proj") +_spawn = [fx.tool_use("toolu_K", "Agent", subagent_type="fork", prompt="p")] +_t4.records = [fx.assistant("req_owner", _spawn, "s", fx.T0, "tool_use"), + fx.user([fx.tool_result("toolu_K", "Fork started")], "r")] +_t4.write_session() +_t4.agent("k1", fx.fork_records("k1", "req_owner", "toolu_K", fx.T0, _spawn), + agentType="fork", toolUseId="toolu_K") +(_fk / "state").mkdir() +for _server in (SERVER_41, SERVER_FULL): + sent, _ = flush_tree(_t4, _server, _fk / "state") + _msgs = [m for a in agent_imports(sent) for m in a["messages"]] + check("fork copies stripped for every server that takes agents", + _msgs and "req_owner" not in _json.dumps(_msgs) + and "fork-context-ref" not in _json.dumps(_msgs)) + +# main() hands the flush whether the session ended. +def _ended_for(payload): + seen = [] + + async def fake_flush(session_id, transcript_path, session_ended=False): + seen.append(session_ended) + + class FakeIn: + def read(self): + return _json.dumps({"session_id": "s1", "transcript_path": __file__, **payload}) + + real_flush, real_log, stdin = fs._flush, fs._log, sys.stdin + fs._flush, fs._log, sys.stdin = fake_flush, (lambda _m: None), FakeIn() + try: + fs.main() + finally: + fs._flush, fs._log, sys.stdin = real_flush, real_log, stdin + return seen + + +check("SessionEnd is a session end", _ended_for({"hook_event_name": "SessionEnd", + "reason": "exit"}) == [True]) +check("a commit is not", _ended_for({"hook_event_name": "PostToolUse", + "tool_input": {"command": "git commit"}}) == [False]) + + print(f"{'FAIL' if FAILURES else 'PASS'}: flush_session") for f in FAILURES: print(f" - {f}") diff --git a/tests/goldens/readers-cli-codex.jsonl b/tests/goldens/readers-cli-codex.jsonl index 82bb300c..e459da42 100644 --- a/tests/goldens/readers-cli-codex.jsonl +++ b/tests/goldens/readers-cli-codex.jsonl @@ -1,6 +1,6 @@ {"conversation_id": "codex-sess-abc", "cwd": "/repo/proj", "git_branch": "synthetic-branch", "host": "codex", "mtime": 1788825600.0, "native_session_id": "sess-abc", "path": "synthetic/rollout-synthetic.jsonl", "source_surface": "Future Desktop", "started_at": "2026-01-01T00:00:00.123456789012Z", "title": "Fix the bug", "type": "session"} -{"cwd": "/repo/proj", "message": {"content": "Fix the bug", "role": "user"}, "timestamp": "2026-01-01T00:00:00Z", "type": "user", "uuid": "b5ebbc9f-91bf-56e7-8227-8c49cb7c854e"} -{"cwd": "/repo/proj", "message": {"content": [{"thinking": "**Planning**", "type": "thinking"}], "model": "gpt-5.3-codex", "role": "assistant"}, "timestamp": "2026-01-01T00:00:00Z", "type": "assistant", "uuid": "ed1cd4ed-6fad-5865-af6d-d3aa34695003"} -{"cwd": "/repo/proj", "message": {"content": [{"text": "On it.", "type": "text"}], "model": "gpt-5.3-codex", "role": "assistant"}, "timestamp": "2026-01-01T00:00:00Z", "type": "assistant", "uuid": "60d6d416-4a30-52e7-9620-a9f57ef951fd"} -{"cwd": "/repo/proj", "message": {"content": [{"id": "call_1", "input": {"cmd": "ls"}, "name": "exec_command", "type": "tool_use"}], "model": "gpt-5.3-codex", "role": "assistant", "usage": {"cache_creation_input_tokens": 10, "cache_read_input_tokens": 60, "input_tokens": 30, "output_tokens": 20}}, "timestamp": "2026-01-01T00:00:00Z", "type": "assistant", "uuid": "9d7c7dfe-8a06-58de-a27f-585fb022f913"} -{"cwd": "/repo/proj", "message": {"content": [{"content": "file.py", "tool_use_id": "call_1", "type": "tool_result"}], "role": "user"}, "timestamp": "2026-01-01T00:00:00Z", "type": "user", "uuid": "b76be0fd-cf17-50a4-8853-6da5c1cfd29a"} +{"cwd": "/repo/proj", "gitBranch": "synthetic-branch", "message": {"content": "Fix the bug", "role": "user"}, "timestamp": "2026-01-01T00:00:00Z", "type": "user", "uuid": "b5ebbc9f-91bf-56e7-8227-8c49cb7c854e"} +{"cwd": "/repo/proj", "gitBranch": "synthetic-branch", "message": {"content": [{"thinking": "**Planning**", "type": "thinking"}], "model": "gpt-5.3-codex", "role": "assistant"}, "timestamp": "2026-01-01T00:00:00Z", "type": "assistant", "uuid": "ed1cd4ed-6fad-5865-af6d-d3aa34695003"} +{"cwd": "/repo/proj", "gitBranch": "synthetic-branch", "message": {"content": [{"text": "On it.", "type": "text"}], "model": "gpt-5.3-codex", "role": "assistant"}, "timestamp": "2026-01-01T00:00:00Z", "type": "assistant", "uuid": "60d6d416-4a30-52e7-9620-a9f57ef951fd"} +{"cwd": "/repo/proj", "gitBranch": "synthetic-branch", "message": {"content": [{"id": "call_1", "input": {"cmd": "ls"}, "name": "exec_command", "type": "tool_use"}], "model": "gpt-5.3-codex", "role": "assistant", "usage": {"cache_creation_input_tokens": 10, "cache_read_input_tokens": 60, "input_tokens": 30, "output_tokens": 20, "output_tokens_details": {"thinking_tokens": 5}, "prompt_tokens": 100}}, "timestamp": "2026-01-01T00:00:00Z", "type": "assistant", "uuid": "9d7c7dfe-8a06-58de-a27f-585fb022f913"} +{"cwd": "/repo/proj", "gitBranch": "synthetic-branch", "message": {"content": [{"content": "file.py", "tool_use_id": "call_1", "type": "tool_result"}], "role": "user"}, "timestamp": "2026-01-01T00:00:00Z", "type": "user", "uuid": "b76be0fd-cf17-50a4-8853-6da5c1cfd29a"} diff --git a/tests/readers_test.py b/tests/readers_test.py index b2abf564..67fcd46f 100644 --- a/tests/readers_test.py +++ b/tests/readers_test.py @@ -159,6 +159,9 @@ def test_codex_transform(): "output_tokens": 20, "cache_read_input_tokens": 60, "cache_creation_input_tokens": 10, + # Reasoning delta from a zero baseline; last request's prompt size. + "output_tokens_details": {"thinking_tokens": 5}, + "prompt_tokens": 100, }, recs[3] print("PASS test_codex_transform") @@ -198,15 +201,218 @@ def test_codex_usage_uses_cumulative_deltas(): "input_tokens": 30, "output_tokens": 20, "cache_read_input_tokens": 60, "cache_creation_input_tokens": 10, + "output_tokens_details": {"thinking_tokens": 5}, + "prompt_tokens": 100, }, assistants[0] + # Reasoning is a cumulative delta too (8 - 5), never the rebroadcast total. assert assistants[1]["message"]["usage"] == { "input_tokens": 30, "output_tokens": 15, "cache_read_input_tokens": 40, "cache_creation_input_tokens": 10, + "output_tokens_details": {"thinking_tokens": 3}, + "prompt_tokens": 180, }, assistants[1] print("PASS test_codex_usage_uses_cumulative_deltas") +def _tc(total, last=None, rate_limits=None, stamp="2026-01-01T00:00:00Z"): + payload = {"type": "token_count", + "info": None if total is None else + {"total_token_usage": total, "last_token_usage": last or total}} + if rate_limits is not None: + payload["rate_limits"] = rate_limits + return {"timestamp": stamp, "type": "event_msg", "payload": payload} + + +def _usage(inp, out, reasoning, cached=0): + return {"input_tokens": inp, "cached_input_tokens": cached, + "cache_write_input_tokens": 0, "output_tokens": out, + "reasoning_output_tokens": reasoning, "total_tokens": inp + out} + + +def _answer(text): + return _line("response_item", {"type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": text}]}) + + +RATE_LIMITS = { + "limit_id": "codex", "limit_name": None, + "primary": {"used_percent": 0.0, "window_minutes": 300, "resets_at": 1792049838}, + "secondary": {"used_percent": 10.0, "window_minutes": 10080, + "resets_at": 1792538546, "account_id": "acct-should-not-leave"}, + "credits": {"has_credits": False, "unlimited": False, "balance": None}, + "individual_limit": {"used_percent": 3.0}, + "spend_control_reached": None, "plan_type": "team", + "rate_limit_reached_type": None, + "access_token": "sk-should-never-leave", +} + + +def test_codex_malformed_reasoning_leaves_counts_unchanged(): + for bad in ("5", True, -1, None, 2.5): + total = _usage(100, 20, 0) + total["reasoning_output_tokens"] = bad + roll = [_line("session_meta", {"id": "r", "cwd": "/x"}), _answer("a"), _tc(total)] + recs, _ = codex.rollout_to_claude_records(roll) + usage = recs[-1]["message"]["usage"] + assert "output_tokens_details" not in usage, (bad, usage) + assert usage["input_tokens"] == 100 and usage["output_tokens"] == 20, usage + # A malformed snapshot breaks the reasoning chain for exactly one step: the + # next delta has no known baseline and is skipped, never invented. + first, second, third = _usage(10, 2, 1), _usage(20, 4, "x"), _usage(30, 6, 9) + roll = [_line("session_meta", {"id": "r", "cwd": "/x"}), + _answer("a"), _tc(first), _answer("b"), _tc(second), + _answer("c"), _tc(third)] + recs, _ = codex.rollout_to_claude_records(roll) + details = [r["message"]["usage"].get("output_tokens_details") for r in recs] + assert details == [{"thinking_tokens": 1}, None, None], details + print("PASS test_codex_malformed_reasoning_leaves_counts_unchanged") + + +def test_codex_inherited_baseline_has_no_first_thinking_delta(): + roll = [_line("session_meta", {"id": "r", "cwd": "/x"}), + _tc(_usage(50, 5, 40)), _answer("a"), _tc(_usage(60, 7, 43))] + recs, _ = codex.rollout_to_claude_records(roll, usage_baseline_unknown=True) + assert recs[-1]["message"]["usage"]["output_tokens_details"] == {"thinking_tokens": 3} + seeded, _ = codex.rollout_to_claude_records( + roll[2:], initial_usage_total=codex._usage_total(_usage(50, 5, 40))) + assert "output_tokens_details" not in seeded[-1]["message"]["usage"], seeded + print("PASS test_codex_inherited_baseline_has_no_first_thinking_delta") + + +def test_codex_prompt_tokens_come_from_the_last_request(): + # One delta covering two requests: summed input 300k, last request 150k. + total = _usage(300_000, 10, 0) + last = _usage(150_000, 5, 0) + roll = [_line("session_meta", {"id": "p", "cwd": "/x"}), _answer("a"), _tc(total, last)] + recs, _ = codex.rollout_to_claude_records(roll) + assert recs[-1]["message"]["usage"]["prompt_tokens"] == 150_000 + for bad in (-1, True, "150000", 2**31): + last = _usage(150_000, 5, 0) + last["input_tokens"] = bad + roll[-1] = _tc(total, last) + recs, _ = codex.rollout_to_claude_records(roll) + assert "prompt_tokens" not in recs[-1]["message"]["usage"], bad + print("PASS test_codex_prompt_tokens_come_from_the_last_request") + + +def test_codex_service_tier_is_carried_when_codex_names_one(): + roll = [_line("session_meta", {"id": "t", "cwd": "/x"}), + _line("event_msg", {"type": "thread_settings_applied", + "thread_settings": {"service_tier": "default"}}), + _answer("a"), _tc(_usage(10, 2, 0)), + _line("turn_context", {"model": "m", "service_tier": "priority"}), + _answer("b"), _tc(_usage(20, 4, 0)), + _line("turn_context", {"model": "m"}), + _answer("c"), _tc(_usage(30, 6, 0)), + _line("event_msg", {"type": "thread_settings_applied", + "thread_settings": {"service_tier": None}}), + _answer("d"), _tc(_usage(40, 8, 0)), + _line("turn_context", {"model": "m", "service_tier": "flex"}), + _line("turn_context", {"model": "m", "service_tier": 7}), + _answer("e"), _tc(_usage(50, 10, 0))] + recs, _ = codex.rollout_to_claude_records(roll) + tiers = [r["message"]["usage"].get("service_tier") for r in recs] + # An absent key keeps the tier; a present one resets it, even to null or + # an unusable value, so a stale tier never lingers on later requests. + assert tiers == ["default", "priority", "priority", None, None], tiers + bare = [roll[0], _answer("a"), _tc(_usage(10, 2, 0))] + recs, _ = codex.rollout_to_claude_records(bare) + assert "service_tier" not in recs[-1]["message"]["usage"] + print("PASS test_codex_service_tier_is_carried_when_codex_names_one") + + +def test_codex_merge_usage_rules(): + record = {"message": {"usage": {"input_tokens": 5, "prompt_tokens": 900, + "output_tokens_details": {"thinking_tokens": 2}, + "service_tier": "flex"}}} + codex._merge_usage(record, {"input_tokens": 3, "prompt_tokens": 400, + "output_tokens_details": {"thinking_tokens": 4, + "bogus": "x"}, + "service_tier": "default", "flag": True}) + assert record["message"]["usage"] == { + "input_tokens": 8, "prompt_tokens": 900, # prompt keeps the max + "output_tokens_details": {"thinking_tokens": 6}, # nested counts add + "service_tier": "default", # text: last wins + }, record + # A type that does not match what is already there is skipped, not coerced. + clash = {"message": {"usage": {"output_tokens_details": 3, "prompt_tokens": "9"}}} + codex._merge_usage(clash, {"output_tokens_details": {"thinking_tokens": 1}, + "prompt_tokens": 4}) + assert clash["message"]["usage"] == {"output_tokens_details": 3, "prompt_tokens": "9"} + text_clash = {"message": {"usage": {"service_tier": 5}}} + codex._merge_usage(text_clash, {"service_tier": "flex"}) + assert text_clash["message"]["usage"] == {"service_tier": 5} + print("PASS test_codex_merge_usage_rules") + + +def test_codex_account_limits_are_allowlisted(): + roll = [_line("session_meta", {"id": "l", "cwd": "/x"}), _answer("a"), + _tc(_usage(10, 2, 0), rate_limits={"plan_type": "plus"}, + stamp="2026-10-14T19:40:10Z"), + # A rate-limit-only update carries no info; it is still the latest. + _tc(None, rate_limits=RATE_LIMITS, stamp="2026-10-14T19:40:11.123Z")] + recs, meta = codex.rollout_to_claude_records(roll) + assert meta["account_limits"] == { + "source": "codex", "observed_at": "2026-10-14T19:40:11.123000Z", + "plan_type": "team", + "primary": {"used_percent": 0.0, "window_minutes": 300, "resets_at": 1792049838}, + "secondary": {"used_percent": 10.0, "window_minutes": 10080, + "resets_at": 1792538546}, + "credits": {"has_credits": False, "unlimited": False, "balance": None}, + "rate_limit_reached_type": None, "limit_id": "codex", "limit_name": None, + "spend_control_reached": None, + }, meta["account_limits"] + assert recs[-1]["message"]["usage"]["input_tokens"] == 10 + _, without = codex.rollout_to_claude_records(roll[:2]) + assert without["account_limits"] is None + print("PASS test_codex_account_limits_are_allowlisted") + + +def test_codex_garbage_rate_limits_never_fail_a_read(): + from readers import codex_limits + base = [_line("session_meta", {"id": "g", "cwd": "/x"}), _answer("a")] + garbage_values = ( + ["not", "a", "dict"], "string", + {"plan_type": 7, "primary": [1], "secondary": {"used_percent": True, + "window_minutes": -5, "resets_at": "soon"}, "credits": "lots", + "limit_id": {"x": 1}, "spend_control_reached": "yes"}, + {"primary": {"used_percent": float("nan")}}, + {"primary": {"used_percent": 101}}, + ) + for garbage in garbage_values: + roll = base + [_tc(_usage(10, 2, 0), rate_limits=garbage)] + for strict in (False, True): # a checked read does not fail either + recs, meta = codex.rollout_to_claude_records(roll, strict=strict) + assert meta["account_limits"] is None, (garbage, meta["account_limits"]) + assert recs[-1]["message"]["usage"]["input_tokens"] == 10 + # An undated snapshot cannot be ordered against a newer one: dropped. + undated = base + [_tc(_usage(10, 2, 0), rate_limits={"plan_type": "team"})] + undated[-1].pop("timestamp") + _, meta = codex.rollout_to_claude_records(undated, strict=True) + assert meta["account_limits"] is None, meta + assert codex_limits.sanitize({"plan_type": "team"}, "yesterday") is None + # Parity with the backend sanitizer: empty strings are dropped, and a + # snapshot whose surviving keys are all null is no snapshot. + stamp = "2026-10-14T19:40:11Z" + assert codex_limits.sanitize({"plan_type": "", "limit_id": ""}, stamp) is None + assert codex_limits.sanitize({"limit_name": None, + "spend_control_reached": None}, stamp) is None + kept = codex_limits.sanitize({"plan_type": "team", "limit_name": None}, stamp) + assert kept["plan_type"] == "team" and kept["limit_name"] is None, kept + # The parser itself never raises, whatever it is handed. + class Hostile(dict): + def get(self, *_a, **_k): + raise RuntimeError("boom") + assert codex_limits.snapshot_from(Hostile(plan_type="x"), "2026-10-14T19:40:11Z") is None + # Equal instants in different spellings canonicalise identically. + assert (codex_limits.canonical_observed_at("2026-10-14T19:40:11Z") + == codex_limits.canonical_observed_at("2026-10-14T21:40:11.000000+02:00") + == codex_limits.canonical_observed_at("2026-10-14T19:40:11.000000000Z") + == "2026-10-14T19:40:11.000000Z") + print("PASS test_codex_garbage_rate_limits_never_fail_a_read") + + def test_codex_recovers_wrapped_ask_without_shifting_legacy_ids(): wrapped = ( "# AGENTS.md instructions\n\n" @@ -300,6 +506,7 @@ def test_codex_usage_without_assistant_gets_stable_record(): "input_tokens": 4, "output_tokens": 2, "cache_read_input_tokens": 8, "cache_creation_input_tokens": 0, + "prompt_tokens": 12, }, usage_record assert usage_record["uuid"] == second[-1]["uuid"] @@ -343,12 +550,14 @@ def test_codex_second_usage_without_output_does_not_hit_prior_assistant(): "input_tokens": 6, "output_tokens": 2, "cache_read_input_tokens": 4, "cache_creation_input_tokens": 0, + "prompt_tokens": 10, }, assistants[0] assert assistants[1]["message"]["content"] == [{"type": "text", "text": ""}] assert assistants[1]["message"]["usage"] == { "input_tokens": 5, "output_tokens": 1, "cache_read_input_tokens": 3, "cache_creation_input_tokens": 0, + "prompt_tokens": 18, }, assistants[1] print("PASS test_codex_second_usage_without_output_does_not_hit_prior_assistant") @@ -369,6 +578,29 @@ def test_codex_missing_call_id_orphans_uniquely(): print("PASS test_codex_missing_call_id_orphans_uniquely") +def test_codex_records_carry_the_session_branch(): + """Codex names the branch once, in ``session_meta.payload.git``; every + record carries it as Claude's top-level ``gitBranch`` — the key the + server's ``branches_from_records`` reads — and nothing else changes.""" + plain, _ = codex.rollout_to_claude_records(CODEX_SYNTH) + meta = dict(CODEX_SYNTH[0], payload={**CODEX_SYNTH[0]["payload"], + "git": {"branch": " feat/x ", + "commit_hash": "abc123"}}) + branched, _ = codex.rollout_to_claude_records([meta, *CODEX_SYNTH[1:]]) + assert branched and all(r.get("gitBranch") == "feat/x" for r in branched), branched + assert not any("commit_hash" in json.dumps(r) for r in branched) + # Identity is untouched: the same uuids, the same records otherwise. + assert [{k: v for k, v in r.items() if k != "gitBranch"} for r in branched] == plain + assert not readers.validate_canonical(branched) + assert not any("gitBranch" in r for r in plain), "absent stays absent" + for git in ({"branch": " "}, {"branch": 7}, {"branch": "b" * 256}, "main", None, {}): + roll = [dict(CODEX_SYNTH[0], payload={**CODEX_SYNTH[0]["payload"], "git": git}), + *CODEX_SYNTH[1:]] + recs, _ = codex.rollout_to_claude_records(roll) + assert not any("gitBranch" in r for r in recs), git + print("PASS test_codex_records_carry_the_session_branch") + + def test_clean_user_text(): assert codex.clean_user_text("# AGENTS.md instructions for /x\n...") is None assert codex.clean_user_text("\n") is None @@ -512,7 +744,7 @@ def _make_cursor_transcript(base: Path, def _make_cursor_store(base: Path, uuid: str = "11111111-2222-3333-4444-555555555555", cwd: str = "/repo/proj", schema_version: int = 1, - node_clocks: bool = True) -> Path: + node_clocks: bool = True, git_branch=None) -> Path: """Build a store.db + meta.json mirroring the observed format: JSON leaf blobs id'd by sha256, an interior node, and a root whose protobuf field 1 lists child hashes in order.""" @@ -520,10 +752,12 @@ def _make_cursor_store(base: Path, uuid: str = "11111111-2222-3333-4444-55555555 import sqlite3 as sq d = base / "19ee0000deadbeef" / uuid d.mkdir(parents=True) - (d / "meta.json").write_text(json.dumps({ - "schemaVersion": schema_version, "cwd": cwd, - "createdAtMs": 1787116704338, "updatedAtMs": 1787116723901, - "hasConversation": True}), encoding="utf-8") + meta = {"schemaVersion": schema_version, "cwd": cwd, + "createdAtMs": 1787116704338, "updatedAtMs": 1787116723901, + "hasConversation": True} + if git_branch is not None: + meta["gitBranch"] = git_branch + (d / "meta.json").write_text(json.dumps(meta), encoding="utf-8") def blob(data: bytes) -> tuple[str, bytes]: return hashlib.sha256(data).hexdigest(), data @@ -644,6 +878,24 @@ def test_cursor_store_without_node_clocks_stays_unmeasured(): print("PASS test_cursor_store_without_node_clocks_stays_unmeasured") +def test_cursor_store_records_carry_the_meta_branch(): + """A store.db session's meta.json names the branch; every record carries + it as top-level ``gitBranch``. No branch in meta.json, or a hook + transcript (which has no meta.json), carries none.""" + with tempfile.TemporaryDirectory() as td: + recs, _ = cursor.to_canonical(_make_cursor_store(Path(td), git_branch=" fix/y ")) + assert recs and all(r.get("gitBranch") == "fix/y" for r in recs), recs + assert not readers.validate_canonical(recs) + for absent in (None, "", " ", "b" * 256): + with tempfile.TemporaryDirectory() as td: + recs, _ = cursor.to_canonical(_make_cursor_store(Path(td), git_branch=absent)) + assert not any("gitBranch" in r for r in recs), absent + with tempfile.TemporaryDirectory() as td: + recs, _ = cursor.to_canonical(_make_cursor_transcript(Path(td)), cwd="/repo/proj") + assert recs and not any("gitBranch" in r for r in recs) + print("PASS test_cursor_store_records_carry_the_meta_branch") + + def test_cursor_transcript_reader_end_to_end(): with tempfile.TemporaryDirectory() as td: path = _make_cursor_transcript(Path(td)) diff --git a/tests/redact_test.py b/tests/redact_test.py index d6cae85e..8f3341d4 100644 --- a/tests/redact_test.py +++ b/tests/redact_test.py @@ -60,11 +60,86 @@ def test_ordinary_content_survives(): ("prefix mentioned in docs", "keys look like mhk_… in the guide"), ("prefix with a short tail", "mhk_short"), ("a path", "/Users/x/.config/memhub-plugin/pak-api.host.json"), - ("base64ish but unprefixed", "ZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNiJ9"), + ("a full git sha, in text that is not a transcript record", + "merged at 6285d40179074934f43deba53713a7d286cb3d5f"), + ("an e-mail address, in text that is not a transcript record", "ask dana@example.com"), + ("a lockfile digest", "integrity sha512-Zk9xQ2pL7vN3mR8tY1wB5cD0eF6gH4iJZk9xQ2pL7vN3mR8tY1wB5cD0eF6gH4iJ"), + ("a long identifier made of words", "MemHubBackendCaptureRedactionModule2026"), + ("code that names a token", "token = self.token"), ]: check(label, r.redact_text(text), text) +# The server's rules, ported (MemHub-Backend app/core/redaction.py). Values +# are made up; each has the shape of the real thing. +VENDOR = [ + ("an Anthropic key", "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", "sk-ant-"), + ("a GitHub token", "ghp_" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8", "ghp_"), + ("an AWS access key id", "AKIAIOSFODNN7EXAMPLE", "AKIA"), + ("a Slack token", "xoxb-123456789012-abcdefghijkl", "xoxb-"), +] + + +def test_the_servers_rules_run_on_this_machine(): + print("\nthe server's rules run here, before upload") + for label, secret, crumb in VENDOR: + out = r.redact_text(f"export KEY={secret} done") + check(f"{label}: value gone", secret in out, False) + out = r.redact_text("use sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789 here") + check("a vendor key keeps its prefix as a breadcrumb", "sk-ant-" in out, True) + jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk" + check("a JWT", r.redact_text(f"Authorization: Bearer {jwt}"), "Authorization: Bearer ") + check("a DSN password", + r.redact_text("postgres://app:hunter2secret@db.internal:5432/prod"), + "postgres://app:@db.internal:5432/prod") + check("an ENV-style secret assignment", + r.redact_text("AUTH_JWT_SECRET=correct-horse-battery"), "AUTH_JWT_SECRET=") + pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAx7Zk9xQ2pL7vN3mR8tY1wB5c\n-----END RSA PRIVATE KEY-----" + out = r.redact_text(pem) + check("a PEM private key body", "MIIEowIBAAKCAQEA" in out, False) + check("…its BEGIN and END lines stay", out.startswith("-----BEGIN RSA PRIVATE KEY-----") + and out.endswith("-----END RSA PRIVATE KEY-----"), True) + bare = "Zk9xQ2pL7vN3mR8tY1wB5cD0eF6gH4iJ" # no prefix, no name: entropy alone + check("a bare high-entropy token (Shannon entropy)", r.redact_text(f"the key is {bare}"), + "the key is ") + check("entropy is what decides", r.shannon_entropy(bare) >= 4.0 + and r.shannon_entropy("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") == 0.0, True) + for text in ["AUTH_JWT_SECRET=correct-horse-battery", f"k {bare}", pem, + "postgres://app:hunter2secret@db.internal:5432/prod", f"Bearer {jwt}"]: + once = r.redact_text(text) + check(f"idempotent: {text[:24]!r}", r.redact_text(once), once) + + +def test_transcript_records_get_capture_strength(): + print("\ntranscript records: capture strength, identifiers kept") + dd = "0123456789abcdef0123456789abcdef" # a Datadog key is 32 hex + rec = {"type": "user", "uuid": "6285d40179074934f43deba53713a7d2", + "sessionId": "0123456789abcdef0123456789abcdef", "cwd": "/Users/dana/repo", + "gitBranch": "fm/topic", "timestamp": "2026-11-10T00:00:00Z", + "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_01AbCdEfGhIjKlMnOpQrStUv", + "content": f"DD_API_KEY={dd}\nbare {dd}\nmail dana@example.com\ncard 4111 1111 1111 1111"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJ"}}]}} + out = r.redact_records([rec])[0] + text = out["message"]["content"][0]["content"] + check("a named hex key", f"DD_API_KEY={dd}" in text, False) + check("a bare 32-hex token", dd in text, False) + check("an e-mail address", "dana@example.com" in text, False) + check("a card number", "4111 1111 1111 1111" in text, False) + check("the record's uuid is untouched", out["uuid"], rec["uuid"]) + check("a 32-hex session id is untouched", out["sessionId"], rec["sessionId"]) + check("cwd and branch are untouched", (out["cwd"], out["gitBranch"]), (rec["cwd"], rec["gitBranch"])) + check("the tool-call id is untouched", out["message"]["content"][0]["tool_use_id"], + rec["message"]["content"][0]["tool_use_id"]) + check("image bytes are untouched", out["message"]["content"][1]["source"]["data"], + rec["message"]["content"][1]["source"]["data"]) + check("idempotent, so the server's pass changes nothing", r.redact_records([out])[0], out) + clean = {"type": "assistant", "uuid": "u1", "message": {"role": "assistant", "content": "ran the tests, 12 passed"}} + check("a clean record comes back equal", r.redact_records([clean])[0], clean) + check("the input is not mutated", rec["message"]["content"][0]["content"].startswith("DD_API_KEY=0123"), True) + + def test_structure_is_preserved(): print("\nstructure is preserved") record = { @@ -148,6 +223,8 @@ def test_identities_for_a_model_bound_window(): if __name__ == "__main__": for test in (test_secrets_are_removed, test_ordinary_content_survives, + test_the_servers_rules_run_on_this_machine, + test_transcript_records_get_capture_strength, test_structure_is_preserved, test_never_raises, test_batch, test_identities_for_a_model_bound_window): test() diff --git a/tests/rulebook_container_test.py b/tests/rulebook_container_test.py index c3434029..2fbe7d20 100644 --- a/tests/rulebook_container_test.py +++ b/tests/rulebook_container_test.py @@ -443,10 +443,9 @@ def test_delivery_lanes() -> None: # The advise-feedback hint sits between the rules and the disclosure; # like the disclosure it carries no book facts and comes from the # hook's own constant. - check("no book facts → byte-identical to the pre-container output, plus the disclosure", + check("no book facts → byte-identical to the pre-container output, plus the notes", ctx(out_old) == pre_container + "\n" + hook.ADVISE_FEEDBACK_HINT + "\n" - + hook.disclosure_instruction( - ["📏 Rule fired: Pair advisory", "📏 Rule fired: Team advisory"]), + + hook.CREDIT_NOTE, repr(ctx(out_old))) # --- an anchor rule is not displaced by a wider book ---------------- diff --git a/tests/rulebook_disclosure_test.py b/tests/rulebook_disclosure_test.py index f3cef254..8cb82b4a 100644 --- a/tests/rulebook_disclosure_test.py +++ b/tests/rulebook_disclosure_test.py @@ -173,49 +173,27 @@ def _repo(td: str) -> tuple[Path, str]: return repo_dir, "disclose-test" -def test_an_advisory_fire_leads_with_the_disclosure_line(): +def test_an_advisory_fire_is_shown_to_nobody_and_goes_to_the_agent_alone(): + """Only a block is put on the person's screen. An advisory fire reaches the + agent as its bullet, with a note that it needs no announcement and may be + credited in a sentence when it changed what the agent did.""" with tempfile.TemporaryDirectory() as td: base = Path(td) / "rb" repo_dir, repo = _repo(td) - seed(base, repo, ["adv"]) + seed(base, repo, ["adv", "adv2"]) out = run(base, repo, "pre", {"cwd": str(repo_dir), "session_id": "s1", "tool_name": "Bash", "tool_input": {"command": "advisory-cmd"}}) - msg = out.get("systemMessage", "") - lines = msg.split("\n") - check("the first line is the disclosure", - lines[:1] == ["📏 Rule fired: Run the suite before pushing"], repr(msg)) - check("the second line is today's branded detail, indented", - len(lines) > 1 and lines[1].startswith(" XTrace ▸ [Run the suite") - and "a red main blocks everyone" in lines[1], repr(msg)) + check("nothing is shown to the user", "systemMessage" not in out, repr(out)) ctx = out["hookSpecificOutput"]["additionalContext"] check("the agent context still carries the rule bullet", "- **[Run the suite before pushing]**" in ctx, ctx) - check("…and ends with the echo instruction", - ctx.rstrip().endswith("needs to hear about._"), ctx[-200:]) - check("…which forbids paraphrasing and omitting", - "Do not paraphrase" in ctx and "did not change what you were going to do" in ctx) - - -def test_the_echoed_lines_are_string_equal_to_what_the_terminal_shows(): - with tempfile.TemporaryDirectory() as td: - base = Path(td) / "rb" - repo_dir, repo = _repo(td) - seed(base, repo, ["adv", "adv2"]) - out = run(base, repo, "pre", {"cwd": str(repo_dir), "session_id": "s1", - "tool_name": "Bash", - "tool_input": {"command": "advisory-cmd"}}) - msg, ctx = out.get("systemMessage", ""), \ - out["hookSpecificOutput"]["additionalContext"] - # Every OTHER line of the systemMessage is a disclosure; the ones - # between are the indented detail lines. - shown = [l for l in msg.split("\n") if not l.startswith(" ")] - quoted = [l for l in ctx.split("\n") if l.startswith(("📏 ", "⛔️ "))] - check("two rules → two stanzas", len(shown) == 2, repr(msg)) - check("the echoed lines equal the shown ones, exactly", - shown == quoted, f"{shown!r} != {quoted!r}") - check("…and the existing order is preserved", - shown[0].endswith("Run the suite before pushing"), repr(shown)) + check("…and no line to echo", + "Rule fired:" not in ctx and "Begin your next reply" not in ctx, ctx[-400:]) + check("…and ends with the credit note", + ctx.rstrip().endswith("tells them why._") + and "team rule from the MemHub Rulebook" in ctx, ctx[-300:]) + check("…which asks for no announcement", "need no announcement" in ctx, ctx[-300:]) def test_a_blocked_gate_discloses_with_the_stop_symbol_and_still_denies(): @@ -268,15 +246,15 @@ def test_session_start_introduces_the_rules_and_discloses_nothing(): ctx.startswith("## 📏 Rulebook (team rules — advisory)"), ctx[:80]) check("the preamble says rules carry CLAUDE.md's standing", "same weight as this repo's CLAUDE.md" in ctx, ctx[:600]) - check("…and mandates the disclosure shape", - "📏 Rule fired: " in ctx, ctx[:600]) + check("…and asks for a credit when a rule changed the outcome, never an announcement", + "They need no announcement" in ctx + and "team rule from the MemHub Rulebook" in ctx, ctx[:600]) check("the posture bullet is unchanged", "- Keep pull requests under 500 lines." in ctx, ctx) check("the armed-rules line is unchanged", "1 rule armed for this repo" in ctx, ctx) - check("NO posture rule is dressed as a fire", - "Rule fired:" not in ctx.replace( - "📏 Rule fired: ", ""), ctx) + check("NO posture rule is dressed as a fire, and no fire line is asked for", + "Rule fired:" not in ctx, ctx) check("session start reaches the agent only, not the terminal", "systemMessage" not in out, str(out)) @@ -304,6 +282,11 @@ def test_book_path_is_answered_by_the_hook_that_reads_it(): env={**os.environ, "PYTHONUTF8": "1"}) check(f"an empty repo argument exits non-zero ({empty!r})", proc.returncode != 0 and proc.stdout.strip() == "", proc.stdout) + proc = subprocess.run([sys.executable, str(HOOK), "book-path", "--projectless"], + capture_output=True, text=True, timeout=60, + env={**os.environ, "PYTHONUTF8": "1"}) + check("--projectless prints the book a call in no checkout reads (repo=\"\")", + proc.returncode == 0 and proc.stdout.strip() == hook.book_path(""), proc.stdout + proc.stderr) proc = subprocess.run([sys.executable, str(HOOK), "book-path"], capture_output=True, text=True, timeout=60, env={**os.environ, "PYTHONUTF8": "1"}) diff --git a/tests/rulebook_hook_test.py b/tests/rulebook_hook_test.py index 92fb673c..5fbfd0cf 100644 --- a/tests/rulebook_hook_test.py +++ b/tests/rulebook_hook_test.py @@ -1988,13 +1988,12 @@ def outj(out): rc, out = run("pre", dict(base, tool_input={"command": "advisory-cmd"}), genv) j = outj(out) - check("advisory: the user's first line is the disclosure, the branded detail beneath it", - j.get("systemMessage", "").startswith("📏 Rule fired: ") - and "\n XTrace ▸ [adv] " in j.get("systemMessage", "") - and "Advisory text" in j["systemMessage"], out) - check("advisory: agent context header stays branded and carries the echo instruction", - "XTrace Rulebook" in ctx(out) and "📏 Rule fired: " in ctx(out) - and "Begin your next reply" in ctx(out), ctx(out)) + check("advisory: nothing is shown to the user", + "systemMessage" not in j, out) + check("advisory: agent context header stays branded, carries the rule and the credit note, no echo", + "XTrace Rulebook" in ctx(out) and "Advisory text" in ctx(out) + and "Rule fired: " not in ctx(out) and "Begin your next reply" not in ctx(out) + and "team rule from the MemHub Rulebook" in ctx(out), ctx(out)) check("advisory: never blocks", "permissionDecision" not in j["hookSpecificOutput"]) push = dict(base, tool_input={"command": "git push --force origin main"}) @@ -2387,6 +2386,7 @@ def denied(cmd): diff_base_checks() spec_pending_probe_checks() armed_lane_checks() + command_armed_checks() min_hook_version_checks() prompt_lane_checks() edit_added_checks() @@ -2406,6 +2406,146 @@ def denied(cmd): +def command_armed_checks() -> None: + """`armed_by_events: ["command"]`: a Bash command matching + `armed_by_command_rx` arms the obligation — on PostToolUse, and only when + the call did not fail — `required_command_rx` clears it and + `gated_command_rx` is where it is collected. Driven through the real hook + process, the way Claude Code and Codex call it.""" + with tempfile.TemporaryDirectory() as td: + repo = os.path.join(td, "cmdrepo") + os.makedirs(repo) + assert _git(repo, "-c", "init.defaultBranch=main", "init", "-q").returncode == 0 + rule = { + "id": "tests-after-move", "on": "ordering", "repo_scope": "any", + "mode": "gate", "_label": "tests-after-move", + "ordering": {"armed_by_events": ["command"], + "armed_by_command_rx": r"\bgit\s+(rm|mv)\b", + "required_command_rx": r"\bpytest\b", + "gated_command_rx": r"\bgit\s+push\b", + "display_name": "the tests"}, + "text": "After git mv/rm, run the tests before you push", "why": "w"} + # The same obligation with a command-position gated pattern, the style + # app/services/rulebook/spec_rule.py writes. + anchored = dict(rule, id="tests-after-move-anchored", _label="tests-after-move-anchored", + ordering=dict(rule["ordering"], + gated_command_rx=r"(?:^|[;&|(]\s*)git\s+push\b")) + seed_book(td, "cmdrepo", [rule]) + n = [0] + + def env(): + # a fresh state dir per scenario: the obligation is per checkout + n[0] += 1 + d = os.path.join(td, f"base{n[0]}") + os.makedirs(d) + shutil.copytree(os.path.join(td, "book"), os.path.join(d, "book")) + return {"MEMHUB_RULEBOOK_BASE": d, "MEMHUB_RULEBOOK_FETCH": "0"} + + def pre(e, command): + n[0] += 1 + return run("pre", {"session_id": "c1", "cwd": repo, "tool_name": "Bash", + "tool_use_id": f"c{n[0]}", + "tool_input": {"command": command}}, e)[1] + + def post(e, command, resp=None, failure=None): + n[0] += 1 + payload = {"session_id": "c1", "cwd": repo, "tool_name": "Bash", + "tool_use_id": f"c{n[0]}", "tool_input": {"command": command}, + "tool_response": resp if resp is not None else {"stdout": "", "exit_code": 0}} + if failure is not None: + payload.update(hook_event_name="PostToolUseFailure", error=failure) + payload.pop("tool_response") + run("post", payload, e) + + e = env() + out = pre(e, "git push") + check("command-armed: nothing is armed before the arming command runs", out.strip() == "", out) + post(e, "git mv old.py new.py") + out = pre(e, "git push origin HEAD") + check("command-armed: a successful `git mv` arms the gate", "BLOCKED" in ctx(out), out) + check("command-armed: the reason names the arming command", + "ran `git mv`" in ctx(out), out) + check("command-armed: the fire is disclosed on systemMessage", + "Rule fired: tests-after-move" in (json.loads(out).get("systemMessage") or ""), out) + ledger = [os.path.join(dp, f) for dp, _, fs in os.walk(e["MEMHUB_RULEBOOK_BASE"]) + for f in fs if "ledger" in dp or "ledger" in f] + check("command-armed: the fire is written to the local ledger", + any("tests-after-move" in open(x, encoding="utf-8", errors="replace").read() + for x in ledger), str(ledger)) + post(e, "pytest -q", failure="Exit code 1\n1 failed, 3 passed") + out = pre(e, "git push") + check("command-armed: a FAILED required command (exit 1) does not clear it", + "BLOCKED" in ctx(out), out) + post(e, "pytest -q", resp={"stdout": "3 passed", "exit_code": 0}) + out = pre(e, "git push") + check("command-armed: a passing required command clears it", out.strip() == "", out) + + e = env() + post(e, "git rm gone.py", resp={"stdout": ""}) # Claude Code: no exit_code on success + out = pre(e, "git push") + check("command-armed: a success without an exit code (Claude Code's payload) arms", + "BLOCKED" in ctx(out), out) + + e = env() + post(e, "git rm missing.py", + failure="Exit code 128\nfatal: pathspec 'missing.py' did not match any files") + out = pre(e, "git push") + check("command-armed: a FAILED arming command (PostToolUseFailure) arms nothing", + out.strip() == "", out) + + e = env() + post(e, "git rm missing.py", resp={"stdout": "", "exit_code": 1}) + out = pre(e, "git push") + check("command-armed: a non-zero exit code arms nothing", out.strip() == "", out) + + e = env() + for mention in ("echo 'git mv a b'", "git commit -m 'git rm the old file'", + "true # git mv a b", "cat <<'EOF' > notes.txt\ngit rm x\nEOF"): + post(e, mention) + out = pre(e, "git push") + check("command-armed: a quoted, commented or heredoc mention arms nothing", + out.strip() == "", out) + + for cmd in ("git mv a.py b.py; ls", "git rm -r old || true", "git mv a.py b.py | cat"): + e = env() + post(e, cmd, resp={"stdout": ""}) + out = pre(e, "git push") + check(f"command-armed: `{cmd}` that did not fail arms, though its exit status " + "is not the move's", "BLOCKED" in ctx(out), out) + + e = env() + post(e, "git mv a.py b.py; pytest -q; ls") + out = pre(e, "git push") + check("command-armed: a required command the exit status does not vouch for " + "clears nothing", "BLOCKED" in ctx(out), out) + + e = env() + post(e, "git mv a.py b.py && pytest -q") + out = pre(e, "git push") + check("command-armed: arming then the required command in one chain leaves " + "nothing owed", out.strip() == "", out) + + e = env() + out = pre(e, "git mv a.py b.py && git push") + check("command-armed: one call that arms and then pushes is gated", + "BLOCKED" in ctx(out) and "this call runs `git mv`" in ctx(out), out) + out = pre(e, "git mv a.py b.py && pytest -q && git push") + check("command-armed: ... unless the required command runs between them", + out.strip() == "", out) + for cmd in ("git mv a.py b.py && git commit -m 'git push later'", + 'git mv a.py b.py && echo "git push"'): + out = pre(e, cmd) + check(f"command-armed: `{cmd}` only MENTIONS the gated command", out.strip() == "", out) + + # A command-position gated pattern sees the push after the separator. + e = env() + seed_book(e["MEMHUB_RULEBOOK_BASE"], "cmdrepo", [anchored]) # this scenario's book only + for cmd in ("git mv a.py b.py && git push", "git mv a.py b.py; git push"): + out = pre(e, cmd) + check(f"command-armed: `{cmd}` is gated under a command-position pattern", + "BLOCKED" in ctx(out) and "this call runs `git mv`" in ctx(out), out) + + def armed_lane_checks() -> None: """An obligation can be armed by something that is not an edit. @@ -3612,9 +3752,10 @@ def fires(rid): check("prompt: a wrapped wake-up fires the rule, on the UserPromptSubmit envelope", hso.get("hookEventName") == "UserPromptSubmit" and "[A loop tick says what it is for]" in hso.get("additionalContext", ""), out) - check("prompt: …and discloses it on both channels", - "📏 Rule fired: A loop tick says what it is for" in doc.get("systemMessage", "") - and "📏 Rule fired: A loop tick says what it is for" in hso.get("additionalContext", ""), + check("prompt: …shown to nobody, with the credit note for the agent", + "systemMessage" not in doc + and "Rule fired:" not in hso.get("additionalContext", "") + and "team rule from the MemHub Rulebook" in hso.get("additionalContext", ""), out) check("prompt: …and never denies anything", "permissionDecision" not in hso, out) rows = fires("loop-tick") @@ -3633,7 +3774,8 @@ def fires(rid): out = prompt("p4", "please deploy now") doc = json.loads(out) if out.strip() else {} check("prompt: a gate-mode prompt rule advises and never blocks", - "Rule fired: prompt-gate" in doc.get("systemMessage", "") + "[prompt-gate]" in (doc.get("hookSpecificOutput") or {}).get("additionalContext", "") + and "systemMessage" not in doc and "permissionDecision" not in (doc.get("hookSpecificOutput") or {}) and "BLOCKED" not in out and [x["mode"] for x in fires("prompt-gate")] == ["advise"], out) diff --git a/tests/spec_owns_port_test.py b/tests/spec_owns_port_test.py index 0d8743d4..be9b2151 100644 --- a/tests/spec_owns_port_test.py +++ b/tests/spec_owns_port_test.py @@ -127,7 +127,8 @@ def git(*args): output = run("pre", {"cwd":tmp,"session_id":"many","tool_name":"Bash","tool_input":{"command":"git push"}}, {"MEMHUB_RULEBOOK_BASE":str(cache),"MEMHUB_RULEBOOK_FETCH":"0"})[1] out = json.loads(output) context = out['hookSpecificOutput']['additionalContext'] - user = out['systemMessage'] + # An advisory fire is shown to nobody: the bullet is the agent's copy, and the only one. + assert 'systemMessage' not in out, out # The spec list rides INLINE on the rule's own bullet, capped at 3 with the rest counted. bullet = [l for l in context.split('\n') if l.startswith('- **[many-specs]**')] assert len(bullet) == 1, context @@ -137,11 +138,6 @@ def git(*args): assert not any(' owns' in l and not l.startswith('- **[') for l in context.split('\n')), context # The agent still gets which owned paths each named spec answers for, on the same line. assert all(f'{p} owns app/item' in bullet[0] for p in named), bullet[0] - # The user's systemMessage names the same capped list, one detail line, and no owned-path detail. - detail = [l for l in user.split('\n') if '[many-specs]' in l and '▸' in l] - assert len(detail) == 1 and '(+9 more)' in detail[0], user - assert ' owns ' not in user and 'app/item' not in user, user - assert len(user.split('\n')) == 2, user # ENG-1153: a big shared file co-owned by narrow specs. Once the branch updates the # spec that governs the change, the file's other owners are not flagged for it —