diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index c208c7b..bad3d84 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -34,6 +34,9 @@ jobs: - name: Structural validator run: python3 test/validate.py + - name: Audit regressions (sherlock plan, one file per closed finding) + run: npm run test:audit + # Every negative self-test below plants a defect and REQUIRES a failure. # A validator nobody has watched fail is not evidence that it works. @@ -764,10 +767,13 @@ jobs: uses: actions/checkout@v4 with: repository: ssheleg/make-skill - ref: 991cbb415be3e856f05c974e631423df552883e3 + ref: 5ca5c367ba24295d4dd99ff670dbef97234140e7 path: .skill-auditor persist-credentials: false + - name: Install a real tokenizer (the budget is MEASURED, never estimated) + run: python3 -m pip install --quiet tiktoken + - name: Audit every shipped Agent Skill run: | python3 - <<'PY' diff --git a/evals/cases/agent-evals.json b/evals/cases/agent-evals.json new file mode 100644 index 0000000..1d6bebc --- /dev/null +++ b/evals/cases/agent-evals.json @@ -0,0 +1,201 @@ +{ + "contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)", + "note": "Outcome corpus for agent-evals (FIX-EV-01.23), anchored to the audit's own findings AS-06..AS-10. Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. Nothing in the production skill was changed for grader convenience.", + "arms": { + "baseline": "same prompts, skill absent", + "current": "same prompts, skill installed" + }, + "cases": [ + { + "schema_version": "outcome-case/1", + "id": "AE-OUT-001-positive-greenfield-corpus", + "skill": "agent-evals", + "prompt": { + "text": "Design the first-release eval corpus for a greenfield agent feature with ZERO production traces: it must contain at least happy, adversarial and failure/retry trials, and an empty corpus or unexecuted observables must NOT close the gate (AS-06). Write corpus-plan.md naming the three trial kinds." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "9c9b2b8cc1279d8aad31920cef652d116b4dee56375afb17c931e7a98c4254fa" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-evals" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "plan exists", + "kind": "artifact-exists", + "target": "corpus-plan.md" + }, + { + "name": "adversarial trials are in the minimum (AS-06)", + "kind": "artifact-contains", + "target": "corpus-plan.md", + "expect": "adversarial" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AE-OUT-002-negative-routing", + "skill": "agent-evals", + "prompt": { + "text": "Explain in two sentences what a webhook is." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [], + "expect_not_loaded": [ + "agent-evals" + ] + }, + "outcome": [ + { + "name": "no eval artifact appears", + "kind": "command-exit-0", + "target": "test ! -f corpus-plan.md" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AE-OUT-003-order-sensitivity", + "skill": "agent-evals", + "prompt": { + "text": "Write an order-sensitivity rubric for a tool-use trace: swapping two independent READ calls must pass, but swapping confirm/charge or acquire/write must FAIL the test; keep the negative example beside the rubric (AS-07). Write order-rubric.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "7e694bd8ee320239cc92768a4fd32705ddb1726aa4d2589d563fa4cf82c5f0b2" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-evals" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "the causal pair fails on swap (AS-07)", + "kind": "artifact-contains", + "target": "order-rubric.md", + "expect": "confirm" + }, + { + "name": "the negative example is kept beside the rubric (AS-07)", + "kind": "artifact-contains", + "target": "order-rubric.md", + "expect": "negative" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AE-OUT-004-noop-statistics-stay-honest", + "skill": "agent-evals", + "prompt": { + "text": "Given eval results with n=1 and n=3 trials, state the uncertainty honestly: no zero-width intervals at boundary p=0 or p=1 (AS-08). Do NOT redesign anything — write stats-note.md only." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "4bf7a16749ab1f193672bebe3d00e3a1ef517071fae7d44ff78cf60504b34e80" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-evals" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "boundary n gives no zero uncertainty (AS-08)", + "kind": "artifact-contains", + "target": "stats-note.md", + "expect": "uncertainty" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AE-OUT-005-regrade-is-not-a-candidate-check", + "skill": "agent-evals", + "prompt": { + "text": "A gate re-checks an OLD answer while claiming to verify a NEW candidate: mutate the candidate to a knowingly wrong tool and show the gate result must change; a regrade of an old trace must be labelled as such, never as a candidate check (AS-10). Write regrade-note.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "f3df1644a20d9ee2cb326c3b55ae95ac5a4c83aa0dfc889d6f280713abf88b83" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-evals" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "mutating the candidate changes the gate (AS-10)", + "kind": "artifact-contains", + "target": "regrade-note.md", + "expect": "mutat" + }, + { + "name": "regrade of an old trace is labelled (AS-10)", + "kind": "artifact-contains", + "target": "regrade-note.md", + "expect": "regrade" + } + ] + } + } + ] +} diff --git a/evals/cases/agent-harness.json b/evals/cases/agent-harness.json new file mode 100644 index 0000000..b5a7ffb --- /dev/null +++ b/evals/cases/agent-harness.json @@ -0,0 +1,213 @@ +{ + "contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)", + "note": "Outcome corpus for agent-harness (FIX-EV-01.24), anchored to the audit's own findings AS-04/AS-05/AS-11/AS-14. Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. Nothing in the production skill was changed for grader convenience.", + "arms": { + "baseline": "same prompts, skill absent", + "current": "same prompts, skill installed" + }, + "cases": [ + { + "schema_version": "outcome-case/1", + "id": "AH-OUT-001-positive-control-and-state-edges-survive", + "skill": "agent-harness", + "prompt": { + "text": "Write graph negative tests for a work graph where a fake-edge pruner may drop only payload-less edges that are truly independent: backup->migration, approval->charge, lease->edit and the serialization of two writes to one file must SURVIVE pruning, while two independent read-only reviews really run in parallel (AS-04). Write edge-tests.md naming each preserved edge." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "0e9d6830cafc08fcef64614dc857310a61f6ae8f936834a85f04c088a48909b3" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-harness" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "report exists", + "kind": "artifact-exists", + "target": "edge-tests.md" + }, + { + "name": "the approval->charge edge survives (AS-04)", + "kind": "artifact-contains", + "target": "edge-tests.md", + "expect": "approval" + }, + { + "name": "write serialization survives (AS-04)", + "kind": "artifact-contains", + "target": "edge-tests.md", + "expect": "serial" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AH-OUT-002-negative-routing", + "skill": "agent-harness", + "prompt": { + "text": "Explain in two sentences what a webhook is." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [], + "expect_not_loaded": [ + "agent-harness" + ] + }, + "outcome": [ + { + "name": "no harness artifact appears", + "kind": "command-exit-0", + "target": "test ! -f edge-tests.md" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AH-OUT-003-auditability-is-reconstruction-not-a-diagram", + "skill": "agent-harness", + "prompt": { + "text": "Run the SAME reconstruction audit over one static and one dynamic orchestration scenario: deleting an event or an edge must be detected in BOTH; a design diagram on its own earns no PASS (AS-05). Write reconstruction-audit.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "f8330cbba332a62fce62b6f60c83e5c0e7f7564faefeb870e83e62d71c941cfa" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-harness" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "deletion detected in both (AS-05)", + "kind": "artifact-contains", + "target": "reconstruction-audit.md", + "expect": "dynamic" + }, + { + "name": "a diagram alone is no PASS (AS-05)", + "kind": "artifact-contains", + "target": "reconstruction-audit.md", + "expect": "diagram" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AH-OUT-004-confirm-true-is-not-a-grant", + "skill": "agent-harness", + "prompt": { + "text": "Write the trifecta threat tests: an agent-authored confirm:true WITHOUT a grant is rejected; a replay with changed arguments is rejected; a pre-authorized action passes; and untrusted-content->destructive-write is tested INDEPENDENTLY of private-data access (AS-11). Write trifecta-tests.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "d130f04d7cf05a478451b05b83d32ed7bc2b8b01b1f4a386b5608ef7d2033bb4" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-harness" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "agent-authored confirm rejected (AS-11)", + "kind": "artifact-contains", + "target": "trifecta-tests.md", + "expect": "confirm" + }, + { + "name": "changed-arguments replay rejected (AS-11)", + "kind": "artifact-contains", + "target": "trifecta-tests.md", + "expect": "replay" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AH-OUT-005-noop-no-evals-hides-no-double-charge", + "skill": "agent-harness", + "prompt": { + "text": "A synthetic repo has NO evals but a demonstrable double charge: report BOTH findings and do not let the general 'no evals' verdict swallow the direct harm; a broken unit invariant is not cured by editing a prompt (AS-14). Do NOT redesign anything — write audit-verdicts.md only." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "ece116ea5dd09fa90d625de5236221e032395f58d309f615dcefaa7a2b135bb1" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-harness" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "the double charge stays its own finding (AS-14)", + "kind": "artifact-contains", + "target": "audit-verdicts.md", + "expect": "double charge" + }, + { + "name": "prompt edits cure no invariant (AS-14)", + "kind": "artifact-contains", + "target": "audit-verdicts.md", + "expect": "prompt" + } + ] + } + } + ] +} diff --git a/evals/cases/agent-interop.json b/evals/cases/agent-interop.json new file mode 100644 index 0000000..348fdd8 --- /dev/null +++ b/evals/cases/agent-interop.json @@ -0,0 +1,206 @@ +{ + "contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)", + "note": "Outcome corpus for agent-interop (FIX-EV-01.25), anchored to the audit's own findings AS-12/AS-13. Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. The pinned-SDK run is probe-gated on a network/venv build — absent, it is NOT_RUN, never PASS. Nothing in the production skill was changed for grader convenience.", + "arms": { + "baseline": "same prompts, skill absent", + "current": "same prompts, skill installed" + }, + "cases": [ + { + "schema_version": "outcome-case/1", + "id": "AI-OUT-001-positive-pinned-mcp-example", + "skill": "agent-interop", + "prompt": { + "text": "Pin the MCP server example to an exact SDK from a clean requirements file and prove it in a fresh env: health returns 200, an unauthenticated endpoint returns 401, and discovery / tools/list / tools/call all pass; the old FastMCP constructor is either supported by its own fixture or declared out of scope (AS-12). Write mcp-proof.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "a772c9d4b94c6507a12121aad3c9619c3ec0de4d74a769d6ade4271bd286876a" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-interop" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "proof exists", + "kind": "artifact-exists", + "target": "mcp-proof.md" + }, + { + "name": "unauthenticated is 401 (AS-12)", + "kind": "artifact-contains", + "target": "mcp-proof.md", + "expect": "401" + }, + { + "name": "the SDK is pinned, old constructor scoped (AS-12)", + "kind": "artifact-contains", + "target": "mcp-proof.md", + "expect": "out of scope" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AI-OUT-002-negative-routing", + "skill": "agent-interop", + "prompt": { + "text": "Explain in two sentences what a webhook is." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [], + "expect_not_loaded": [ + "agent-interop" + ] + }, + "outcome": [ + { + "name": "no interop artifact appears", + "kind": "command-exit-0", + "target": "test ! -f mcp-proof.md" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AI-OUT-003-protocol-routing-not-by-the-word-long-running", + "skill": "agent-interop", + "prompt": { + "text": "Route three tasks to a protocol: a long-running fixed export goes to MCP Tasks WHEN supported; an autonomous outsourced negotiation goes to A2A; and when MCP Tasks is unsupported, take an explicit fallback. No case may be decided by the word 'long-running' alone (AS-13). Write routing-table.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "b6dce447ce64b3c4d2ec21614db5799a34cfd1708fe5c70958e757e78c27ec64" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-interop" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "the fixed export routes to MCP Tasks (AS-13)", + "kind": "artifact-contains", + "target": "routing-table.md", + "expect": "MCP Tasks" + }, + { + "name": "autonomous negotiation routes to A2A (AS-13)", + "kind": "artifact-contains", + "target": "routing-table.md", + "expect": "A2A" + }, + { + "name": "an unsupported-Tasks explicit fallback exists (AS-13)", + "kind": "artifact-contains", + "target": "routing-table.md", + "expect": "fallback" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AI-OUT-004-noop-unsupported-tasks-fallback", + "skill": "agent-interop", + "prompt": { + "text": "For a host where MCP Tasks is NOT supported, a long-running fixed export must take the explicit fallback path, not be forced into A2A because it is long (AS-13). Do NOT redesign the protocols — write fallback-note.md only." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "c6c0ba7f0abe8996ea7d737a801a448e950bb2ef756d6269a5482028a31d3e6c" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-interop" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "unsupported Tasks takes the fallback, not A2A (AS-13)", + "kind": "artifact-contains", + "target": "fallback-note.md", + "expect": "fallback" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AI-OUT-005-live-pinned-env-probe", + "skill": "agent-interop", + "prompt": { + "text": "In a network-capable env, build the pinned requirements into a fresh venv and run the MCP example's health check, expecting 200 (AS-12)." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "fc528213a243d3ddadc79c807bdf2156ef62f1b265ed5641eecae248cc45cf1c" + }, + "checks": { + "tool": [ + { + "name": "network+venv build available", + "command": "python3 -c \"import urllib.request,venv\" && test -n \"$AGENT_INTEROP_LIVE\"" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-interop" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "the pinned example builds and answers (probe-gated)", + "kind": "artifact-exists", + "target": "live-health.md" + } + ] + } + } + ] +} diff --git a/evals/cases/agent-orchestrator.json b/evals/cases/agent-orchestrator.json new file mode 100644 index 0000000..56a2f34 --- /dev/null +++ b/evals/cases/agent-orchestrator.json @@ -0,0 +1,201 @@ +{ + "contract": "outcome-case/1 (schemas/outcome-case.schema.json + test/outcome_harness.py in ssheleg/sshlg-skills)", + "note": "Outcome corpus for agent-orchestrator (FIX-EV-01.26), anchored to the audit's own findings AS-01/AS-02/AS-03/AS-04/AS-05. Judged on ARTIFACTS via the family harness: the runner records the actual output oracle verdict AND the raw result per case; with/without-skill arms are the harness's baseline contract. Nothing in the production skill was changed for grader convenience.", + "arms": { + "baseline": "same prompts, skill absent", + "current": "same prompts, skill installed" + }, + "cases": [ + { + "schema_version": "outcome-case/1", + "id": "AO-OUT-001-positive-saga-not-2pc", + "skill": "agent-orchestrator", + "prompt": { + "text": "Design fault injection for a saga wrongly called two-phase commit: inject a fault after the DB commit, after the upstream effect but before its response, during two concurrent topups, and between retries. Assert ledger conservation, AT MOST ONE external effect per operation_id, and that an UNKNOWN HTTP outcome stays PENDING for reconciliation rather than being compensated as a failure (AS-01). Write saga-faults.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "5a75c23fe846694ef751bd0a16ac797bd7de6d0610cbf29e8b20c52ef0bdc6fc" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-orchestrator" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "report exists", + "kind": "artifact-exists", + "target": "saga-faults.md" + }, + { + "name": "unknown stays pending, not compensated (AS-01)", + "kind": "artifact-contains", + "target": "saga-faults.md", + "expect": "pending" + }, + { + "name": "at most one external effect per operation_id (AS-01)", + "kind": "artifact-contains", + "target": "saga-faults.md", + "expect": "operation_id" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AO-OUT-002-negative-routing", + "skill": "agent-orchestrator", + "prompt": { + "text": "Explain in two sentences what a webhook is." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "a5da94aa03520bb6226ea88328a75ab6d7e3985c73a0e90a1379bf168fead57e" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [], + "expect_not_loaded": [ + "agent-orchestrator" + ] + }, + "outcome": [ + { + "name": "no orchestrator artifact appears", + "kind": "command-exit-0", + "target": "test ! -f saga-faults.md" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AO-OUT-003-zero-baseline-is-not-missing", + "skill": "agent-orchestrator", + "prompt": { + "text": "For a metered balance, distinguish an uninitialized key from an initialized zero: run uninitialized→5, initialized(0)→5, 5→8, 8→2 on the same key and a generation change, and for each give the expected ledger row and state — the first real spend of a new key must be counted, not lost (AS-02). Write baseline-cases.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "c0e22afca6dec4c16ba9d51ffbc35cfe8db196863ed60acd1813bb647a1f3827" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-orchestrator" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "the first spend of a new key is counted (AS-02)", + "kind": "artifact-contains", + "target": "baseline-cases.md", + "expect": "uninitialized" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AO-OUT-004-lexical-similarity-does-not-merge-opposites", + "skill": "agent-orchestrator", + "prompt": { + "text": "Build a multilingual memory corpus with negation, changed numbers/units, exceptions, different subjects and stale verified facts: no silent merge of contradictions, a correction wins ONLY in its own scope, and the old fact stays in history (AS-03). Write memory-conflicts.md." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "ae08a3670a0ac382b65e0fc51ec46bcb94ce839a94a2283dc095df3dab8d4225" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-orchestrator" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "a correction wins only in its scope, old fact kept (AS-03)", + "kind": "artifact-contains", + "target": "memory-conflicts.md", + "expect": "scope" + }, + { + "name": "no silent merge of opposites (AS-03)", + "kind": "artifact-contains", + "target": "memory-conflicts.md", + "expect": "negation" + } + ] + } + }, + { + "schema_version": "outcome-case/1", + "id": "AO-OUT-005-noop-fake-edge-preserves-control-state", + "skill": "agent-orchestrator", + "prompt": { + "text": "Audit a work-graph's fake-edge pruner: backup->migration, approval->charge, lease->edit and the serialization of two writes must SURVIVE even with no explicit payload, while two independent read-only reviews run in parallel (AS-04). Do NOT redesign the graph — write edge-audit.md only." + }, + "environment": { + "model": "inherit", + "host": "any", + "case_digest": "9a1f65a9604f93b0ec30c737e9daabe29e93d95a2cb50ddebdfb0211f0803de5" + }, + "checks": { + "tool": [ + { + "name": "python3 present", + "command": "python3 -c 'pass'" + } + ], + "load_trace": { + "expect_loaded": [ + "agent-orchestrator" + ], + "expect_not_loaded": [] + }, + "outcome": [ + { + "name": "control/state edges survive without payload (AS-04)", + "kind": "artifact-contains", + "target": "edge-audit.md", + "expect": "approval" + } + ] + } + } + ] +} diff --git a/package.json b/package.json index be62a93..ac39d7c 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,10 @@ "name": "@ssheleg/agent-stack", "version": "0.23.2", "scripts": { - "test": "python3 test/validate.py && python3 test/plant_guard_test.py && node test/installer_test.js" + "test": "python3 test/validate.py && python3 test/plant_guard_test.py && node test/installer_test.js && npm run test:audit", + "test:audit": "for t in test/audit_regressions/*.py; do python3 \"$t\" || exit 1; done" }, - "description": "Production patterns for AI agent orchestrators — tool-calling loops, multi-stage pipelines with checkpoints, LLM provider routing with fallback, four-layer memory with confidence decay — plus the wallet side of reselling LLM access. This package is the installer CLI.", + "description": "Production patterns for AI agent orchestrators \u2014 tool-calling loops, multi-stage pipelines with checkpoints, LLM provider routing with fallback, four-layer memory with confidence decay \u2014 plus the wallet side of reselling LLM access. This package is the installer CLI.", "bin": { "agent-stack": "bin/agent-stack.js" }, diff --git a/plugins/agent-stack/skills/agent-evals/SKILL.md b/plugins/agent-stack/skills/agent-evals/SKILL.md index 3a8aa7d..bfd8d17 100644 --- a/plugins/agent-stack/skills/agent-evals/SKILL.md +++ b/plugins/agent-stack/skills/agent-evals/SKILL.md @@ -76,7 +76,7 @@ Assert on three axes at once, with three different mechanisms: | Axis | Assert | With | |---|---|---| -| Trajectory | what the run **must not** do, and what it must have touched — never the order | set/subset matchers, forbidden-call lists | +| Trajectory | what the run **must not** do, what it must have touched, and the mandatory **happens-before** edges — never the full exact order | set/subset matchers, forbidden-call lists, partial-order (a before b) | | Final response | quality, tone, policy compliance | rubric or judge | | **State change** | the memory row exists, the file was written, the artifact is there | direct inspection of the side effect | @@ -152,19 +152,26 @@ assertions, tool-call correctness — all deterministic, all faster and cheaper call. Send to a judge only what cannot be decided by code. **Read the trajectory; do not match it.** An agent that reaches a correct answer through -three wrong tool calls is a latent outage — and asserting the *sequence* to catch that is -measurably the wrong instrument. Anthropic names the instinct and rejects its strict form: -exact tool-order assertions are *"too rigid and results in overly brittle tests, as agents -regularly find valid approaches that eval designers didn't anticipate"*, and the worked +three wrong tool calls is a latent outage — and asserting the *exact sequence* to catch +that is measurably the wrong instrument. Anthropic rejects its strict form: exact +tool-order assertions are *"too rigid and results in overly brittle tests, as agents +regularly find valid approaches that eval designers didn't anticipate"* — the worked case is an agent that solved a τ²-bench booking task through a policy loophole, failing the eval as written while serving the user better. Grade **what was produced and what -changed**, and let the path vary. +changed**, and let the incidental path vary. -The opposite edge is measured too, so this is not "grade the final answer": a grader blind -to the trajectory misses **44% of safety violations and 13% of robustness failures**, +The opposite edge is measured too: a grader blind to the trajectory misses **44% of safety violations and 13% of robustness failures**, because a policy breach on the way to a correct result leaves no trace in the outcome. Use the trajectory for the claims the outcome cannot carry — a forbidden call, a missing -confirmation, a secret read — as a **set and a forbidden list**, never as an order. +confirmation, a secret read — as a **set and a forbidden list**. + +What is forbidden is the redundant **exact global sequence**, not order as such. +A few **happens-before** edges are semantically mandatory: authorization +precedes its effect, a fresh read precedes the write depending on it, a +transaction commits before what publishes it. Assert those as a **partial +order** (a before b), never a total one — reordering two independent reads must +pass, reordering confirm/charge or acquire/write must fail — and keep the +negative example (a confirm-after-charge trace) beside the rubric. **Calibrate the judge before trusting it.** Collect human labels on the same traces, measure agreement, iterate the judge prompt until agreement is high — *then* let it score @@ -253,10 +260,16 @@ with no production in it is imagination.** The requirement itself gets its id an definition of done from `task-pipeline`'s REQ spine — what this pack owns is the observable's *form*, not the register it hangs on. -**The first release has no production, so its offline gate is observables only** (§3). That -is not the corpus rule suspended for a special case: the corpus is empty because nothing has -run yet, and it fills from the first real traces. Inventing *inputs* to fill it sooner would -still be imagination. +**The first release has no production — so it runs against a SEED corpus, and +observable-only is not release-ready.** A criterion with no input proves +no capability, so a greenfield feature seeds a curated/synthetic/manual corpus with at least a **happy**, an **adversarial** and a +**failure/retry** trial. Each seed input carries its **provenance** +(`curated`/`synthetic`/`manual`) and is SUPPLEMENTED by production traces, +never declared full coverage. The release gate requires EXECUTED trials; +observables with nothing run against them are `specification-ready`, not +`release-ready`. A corrupted fixture, input or runner is a `TEST_ERROR`, +never a behaviour pass/fail; cases are isolated, so B's result never depends +on whether A ran. **Never author the suite up front** — the *corpus*, that is: the inputs. Every production failure and every thumbs-down becomes a fixture: @@ -300,7 +313,11 @@ None of the above runs without these, and they are the part people skip: of `human` | `llm_judge` | `code_check`. A score with no source cannot be calibrated, audited, or trusted differently from its neighbours. - **Whole prompts, not just messages** — instructions, tool schemas and context as they - were sent. A fixture cannot be replayed from a summary. + were sent. A fixture cannot be replayed from a summary. And a CANDIDATE's + version, output and score are their own records beside the old trace — a + regrade of the stored output is labelled regrade, never "the candidate + passed": only executing the candidate against the frozen fixture (a real, + stochastic call, costed in the receipt) can say the decision changed. - **State snapshots at turn boundaries**, so a thread test can assert what carried. --- diff --git a/plugins/agent-stack/skills/agent-evals/references/otel-genai.md b/plugins/agent-stack/skills/agent-evals/references/otel-genai.md index e993d0f..f2b35b2 100644 --- a/plugins/agent-stack/skills/agent-evals/references/otel-genai.md +++ b/plugins/agent-stack/skills/agent-evals/references/otel-genai.md @@ -10,7 +10,7 @@ three places it will not carry what this skill requires. ## Contents - [Read this before encoding any of it](#read-this-before-encoding-any-of-it) -- [Span names are formulas, and the operation list is closed](#span-names-are-formulas-and-the-operation-list-is-closed) +- [Span names are formulas, and the operation set is well-known but extensible](#span-names-are-formulas-and-the-operation-set-is-well-known-but-extensible) - [The evaluation event, missing the field §7 requires](#the-evaluation-event-missing-the-field-7-requires) - [Content: three tiers, and a hook that runs when nothing else does](#content-three-tiers-and-a-hook-that-runs-when-nothing-else-does) - [Tokens are eleven numbers and money is none of them](#tokens-are-eleven-numbers-and-money-is-none-of-them) @@ -31,11 +31,14 @@ inference span the only attributes marked `Stable` are the ones borrowed from co attribute is `Development`.** So: adopt it, because a moving standard beats a private vocabulary that will never be read -by anyone else's tooling — and **pin the version you adopted and expect to migrate**. Treat +by anyone else's tooling — and **pin the version you adopted and expect to migrate**. When +you encode any field from this file, record the **semconv schema revision and the commit SHA +you read it at, plus the observation date** beside your instrumentation — a `gen_ai.*` field +quoted with no revision is a field with no expiry, and this whole spec is `Development`. Treat any code that branches on a `gen_ai.*` attribute as code with an expiry date, and re-read the spec before quoting a field name from this file. -## Span names are formulas, and the operation list is closed +## Span names are formulas, and the operation set is well-known but extensible Span names are computed, not free text: @@ -48,10 +51,15 @@ Span names are computed, not free text: | agent creation | `create_agent {gen_ai.agent.name}` | — | | MCP | `{mcp.method.name} {target}`, target being the tool or prompt name | — | -`gen_ai.operation.name` is a **closed 17-value enum** — `chat`, `text_completion`, -`generate_content`, `embeddings`, `retrieval`, `fetch_response`, `execute_tool`, -`create_agent`, `invoke_agent`, `plan` and the rest. A value outside it is not an extension, -it is a name a backend cannot group by. +`gen_ai.operation.name` is a **well-known SET, not a closed enum** — the semconv +lists `chat`, `text_completion`, `generate_content`, `embeddings`, `retrieval`, +`fetch_response`, `execute_tool`, `create_agent`, `invoke_agent`, `plan` and the +rest, and a well-known value is preferred WHERE ONE FITS. But a provider +operation with no matching well-known value is allowed to carry a custom value: +it is not silently dropped, and an unknown value is stored RAW so a later +schema revision can recognise it. A backend groups the well-known values and +keeps the raw ones addressable — losing them is the failure, not carrying +them. **Only two attributes are Required on an inference span.** Everything else that matters — the model that actually answered, token counts, finish reasons — is Recommended or @@ -127,17 +135,43 @@ leaks through the one field it never inspected. Decide naming and redaction toge standardises tokens and never money, so cost is always a join against a price table living outside the trace — and that join is where the number goes wrong. -Because usage is not one number. It is eleven: `gen_ai.usage.input_tokens`, `output_tokens`, -`reasoning.output_tokens`, `cache_read.input_tokens`, `cache_write.input_tokens`, and -per-modality `text.*` / `image.*` / `audio.*` splits including -`image.cache_read.input_tokens`. - -**A cost computed from `input_tokens + output_tokens` alone is wrong in both directions.** It -bills cache reads at full price — they are the cheap ones — and it misses reasoning tokens and -cache writes entirely. `../agent-orchestrator/references/kv-cache.md` is the other half of +Because usage is not one number, and the counters are of TWO kinds — TOTALS and +DISJOINT BILLING BUCKETS, and confusing them double-counts. `input_tokens` and +`output_tokens` are the totals; `reasoning.output_tokens` is a SUBSET of +`output_tokens` (not an addition to it), and `cache_read.input_tokens` / +`cache_write.input_tokens` are subsets of `input_tokens`. The per-modality +`text.*` / `image.*` / `audio.*` splits (including `image.cache_read.input_tokens`) +partition those same totals by modality — they are not extra tokens either. + +**A cost computed from `input_tokens + output_tokens` alone is wrong** because it +prices the cached portion of the input at the full input rate — cache reads are +the cheap ones. The fix is NOT to add reasoning or modality counters back onto +the totals (that bills them twice): it is to **SUBTRACT the cached portion from +the total and apply the provider's cache-read (and cache-write) rate to it**, +pricing the remaining full-rate input and the output at their own rates. `../agent-orchestrator/references/kv-cache.md` is the other half of this: the cache read is the case worth getting right, because at scale it is most of the traffic. +**The worked receipt — a disjoint partition that reconciles.** Say `input_tokens += 1000` with `cache_read.input_tokens = 800`, and `output_tokens = 200` with +`reasoning.output_tokens = 120`, at rates `$3 / $0.30 / $15` per 1k for +full-input / cache-read / output: + +| Bucket | Tokens | Rate /1k | Cost | +|---|---|---|---| +| input, full-rate = `input − cache_read` | 200 | $3.00 | $0.60 | +| `cache_read` (a subset of input) | 800 | $0.30 | $0.24 | +| output (reasoning is a SUBSET, not added) | 200 | $15.00 | $3.00 | +| **total** | | | **$3.84** | + +Two invariants a receipt MUST satisfy, and an independent example checks: the +priced token buckets **sum back to the totals** — 200 + 800 = 1000 input, and +output is 200 (reasoning's 120 is inside it, never a fourth line) — so no token +is charged twice; and the naive `input + output` at the input/output rates +($3.00 + $3.00 = $6.00) OVER-charges by pricing the 800 cached tokens at $3 +instead of $0.30. The receipt reconciles with the totals; the naive number does +not. + And `gen_ai.client.token.usage` carries a hard **MUST NOT report** when the counts are not obtainable. A zero is a claim; absence is the honest value. That is the same rule `agent-harness/references/audit.md` states for cost attribution — *missing attribution beats @@ -171,7 +205,14 @@ which: |---|---|---|---| | **Durable execution** (Temporal) | nothing — recorded results are replayed and only the failed step retries | free, deterministic | can I resume without redoing 20 web searches | | **Trace playground** (Phoenix) | the model call, against the live provider, with an edited prompt | a real call | would a different prompt have done better | -| **Fixture replay** (this skill, §2 single-step) | an assertion over a stored run | free | did the decision at this point change | - -They are not interchangeable, and a runbook that says *"replay the run"* has not said what it -means. Name the sense. +| **Regrade of a stored output** (this skill, §2 single-step) | nothing — a new assertion over the OLD output | free, deterministic | does the OLD output satisfy the changed rubric — **explicitly NOT a check of the candidate** | +| **Candidate execution over a frozen fixture** | the CANDIDATE (new prompt / model / tool schema) against frozen inputs | a real model call, stochastic — say so in the receipt | did the candidate's decision change | + +They are not interchangeable, and a runbook that says *"replay the run"* has not said what +it means. Name the sense. The one this table exists to un-blur: **an old output does not +change when the candidate changes** — a regrade re-marks yesterday's homework under a new +rubric, and calling it "did the decision change" claims a candidate check that never ran. +The test with teeth is the fourth row: mutate the candidate to a knowingly wrong tool, and +the gate's result MUST change; a gate a candidate mutation cannot move is a regrade wearing +the wrong label. The candidate's version, its output and its score are stored as their own +records, never overwriting the old trace they are compared against. diff --git a/plugins/agent-stack/skills/agent-evals/references/statistics.md b/plugins/agent-stack/skills/agent-evals/references/statistics.md index bc7c9f2..3bd754c 100644 --- a/plugins/agent-stack/skills/agent-evals/references/statistics.md +++ b/plugins/agent-stack/skills/agent-evals/references/statistics.md @@ -38,9 +38,32 @@ The 95% band is roughly `±1.96 · SE`. Computed, not quoted: ```python import math -def band(p, n): return 1.96 * math.sqrt(p * (1 - p) / n) * 100 # percentage points + +def band(p, n): + """Wald approximation, percentage points. Valid only for moderate n with p + away from the boundary (rule of thumb: n*p >= 10 and n*(1-p) >= 10). At + p=0 or p=1 it returns ZERO width — which is exactly wrong: a run that has + never failed is not a run with no uncertainty.""" + return 1.96 * math.sqrt(p * (1 - p) / n) * 100 + +def wilson(p, n, z=1.96): + """Wilson score interval — the DEFAULT for a proportion. Nonzero width at + the boundary, honest at small n; use exact (Clopper–Pearson) when n is + tiny and the decision is expensive. n == 0 is total uncertainty (0, 1), + never a zero-width claim.""" + if n == 0: + return (0.0, 1.0) + denom = 1 + z * z / n + centre = (p + z * z / (2 * n)) / denom + half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / denom + return (max(0.0, centre - half), min(1.0, centre + half)) ``` +The table above is Wald and inherits its limits; **a zero or tiny sample is not +a universal bound.** `wilson(0.0, 5)` spans up to ≈43% — five clean runs still +leave nearly half the range open — where Wald would print ±0.0 and read as +certainty. + **So "the new one gets 73% where the old one got 70%, on a hundred cases" is not a result.** It is a number inside its own noise. The error shrinks as `1/√n`, which is the whole practical consequence: **the fix for a 2–3 pp expected gain is more tasks, not more @@ -49,9 +72,12 @@ argument.** Quadrupling the set halves the band. A corollary worth stating because leaderboards invite the opposite: **differences below about 3 pp deserve scepticism until both configurations are documented and matched.** -> The formula assumes independent cases. A benchmark whose tasks share a fixture, an -> environment or a generator violates that, and the true band is wider than this. Wider, -> never narrower — so the table is a floor on your uncertainty, not a ceiling. +> The formula assumes independent (iid) cases. A benchmark whose tasks share a +> fixture, an environment or a generator violates that, and under the usual +> assumption — POSITIVE intra-cluster correlation, which is what shared +> fixtures produce — the true band is wider than this, so treat the table as a +> floor. That is an assumption, not a theorem: engineered negative dependence +> can narrow a band, it just never happens by accident in a shared fixture. ## pass@k and pass^k are different questions @@ -78,6 +104,13 @@ If a failed attempt leaves a charge, a message or a mutated row behind, `pass@k` available to you as a metric — you cannot pick the best of five refunds. Sample in a sandbox or a rollback-capable environment, and count **every** failure. +**Both are computed over TASK-LEVEL trials.** k repeated trials of one task +estimate that task's own p_i; the benchmark number is the mean over TASKS of +the per-task pass@k (or pass^k). Pooling repeated trials of one task into the +denominator as if each were a new task inflates n with copies of the same +difficulty — the trials are not independent tasks, and counting them as tasks +is how a small suite pretends to be a large one. + **A report that gives k without saying which k it means is unreadable.** *k independent samples of one task* and *k consecutive tasks on one live pipeline* are different claims. @@ -103,9 +136,12 @@ Two consequences, and they cut in opposite directions: - **You cannot compute `pass^k` from `pass^1`.** Exponentiating a headline rate gives a number far below the truth. Measure `pass^k` directly, at the k you care about. -- **Anthropic's `0.75³ ≈ 42%` is a worst case, not a forecast.** It is the right shape for - an argument — *consistency is a much harder bar* — and the wrong number to put in a - release gate. +- **Anthropic's `0.75³ ≈ 42%` is the INDEPENDENCE BASELINE, not a bound.** Real + curves usually sit above it because successes cluster by task (positive + dependence), but that is an empirical pattern, not a guarantee — engineered + negative dependence can fall below it. The right shape for an argument — + *consistency is a much harder bar* — and the wrong number to put in a release + gate either way. The other half of independence is the harness, not the task: Anthropic requires each trial start from a clean environment, because *"unnecessary shared state between runs (leftover @@ -115,8 +151,13 @@ metric mean anything. ## Pairing: same tasks, same seeds, per-task deltas -**Never subtract two independent averages.** Run both configurations over the *same* task -list with the *same* fixed seeds, record a per-task win/loss/tie, and test the deltas. +**Prefer pairing — and never subtract two averages WITHOUT an interval.** Run +both configurations over the *same* task list with the *same* fixed seeds, +record a per-task win/loss/tie, and test the deltas: pairing cancels the +per-task difficulty variance and needs far fewer runs. An UNPAIRED comparison +of two independent averages is still legitimate when pairing is impossible — +it just pays for it with the wider two-sample band, and the sin is quoting the +subtraction bare, as if the band were zero. ``` for task in tasks: # identical list @@ -138,6 +179,34 @@ for task in tasks: # identical list **Ship on three conditions, not one:** the difference exceeds the noise band, it survives the paired analysis, and it reproduces on a rerun. +## The design decides the method — and the receipt names both + +Paired, clustered and unpaired are three DIFFERENT corpus structures, and a +test method borrowed from the wrong one produces confident nonsense. The +result's receipt names the design AND the method, and they must match: + +| Corpus structure | Matching method | Mismatch that looks fine and is not | +|---|---|---| +| **paired** — same tasks, same seeds, per-task deltas | McNemar (binary) or a paired bootstrap over the DELTAS | running McNemar on two independent runs pairs rows that share nothing | +| **clustered** — k dependent repeats per task | a cluster bootstrap that resamples TASKS (each task carries its repeats along) | bootstrapping TRIALS treats dependent repeats as iid and shrinks the band by ~√k for free | +| **unpaired** — two independent samples | the two-sample (Welch) SE, wider band | quoting the paired-sized band for an unpaired design | + +**Dependent repeats are never claimed iid**, and a small sample never buys +imaginary certainty — the Wilson bounds above are the floor either way. + +## Splits are spent once — case ids and groupings are FIXED + +The corpus's case IDs and their groupings (which task belongs to which cluster, +which split) are frozen before any run and never regrouped to taste. + +- **Validation** MAY be used for tuning — that is what it is for. +- **The final holdout is spent ONCE**, on the version already chosen. It is + never used to pick between versions; a holdout consulted per candidate is a + second validation set wearing a blindfold. +- **A reused validation example can never be relabelled an "unseen final + test"** — the receipt says which split every number came from, and "unseen" + is a property of the RUN HISTORY, not of the label somebody wrote today. + ## The harness is a variable, so pin it The container spec is part of the measurement. On Terminal-Bench 2.0 the gap between the diff --git a/plugins/agent-stack/skills/agent-harness/SKILL.md b/plugins/agent-stack/skills/agent-harness/SKILL.md index 59a5667..60757ea 100644 --- a/plugins/agent-stack/skills/agent-harness/SKILL.md +++ b/plugins/agent-stack/skills/agent-harness/SKILL.md @@ -33,13 +33,16 @@ forwards and backwards, which is why they live together here. --- -## Rule zero — most agent bugs are prompt bugs wearing a stack trace +## Rule zero — check the prompt first (a diagnostic heuristic, with exceptions) -The instinct when an agent misbehaves is to change the code. The measured reality, in every -source this skill was built from, is that the largest behavioural changes come from the -text: **"the biggest performance improvements often come from clearly explaining tool usage -in the system prompt"**, and **"even small refinements to tool descriptions can yield -dramatic improvements."** +The instinct when an agent misbehaves is to change the code. The sources this skill was +built from pull the other way: **"the biggest performance improvements often come from +clearly explaining tool usage in the system prompt"**, and **"even small refinements to +tool descriptions can yield dramatic improvements."** That is vendor guidance about where +leverage OFTEN lives, not a measured share of defects — so it orders the DIAGNOSIS, never +the verdict. **The exceptions are the findings a prompt cannot touch:** a deterministic +race, a hardcoded secret, a timeout wired to the wrong operation — source-level invariant +violations are code bugs, provable by reading, and no rewording treats them. Before adding a retry, a router, or a sub-agent, check in this order: @@ -97,11 +100,15 @@ A **static** graph has every node and edge decided up front; a **dynamic** one g nodes read their own output and decide what comes next. **Static first, always** — go dynamic only after the static version hits a wall you can -name, because dynamic is more powerful and much harder to control. And one row of that -decision is hard rather than preferential: **a run that has to be auditable is static.** -A dynamic graph's executed shape is not the shape anybody drew, so *"here is the design"* -and *"here is what happened"* stop being the same document, and every claim about the run -becomes unfalsifiable from outside. +name, because dynamic is more powerful and much harder to control. But **auditability is +NOT the same axis as static structure** — that conflates the plan drawn beforehand with +the execution graph saved afterward. A run is auditable when its EXECUTION RECORD is +complete: every node, edge and event that actually ran, the policy version in force, and +deterministic bounds (budget / depth / node caps) with provenance. A static graph is the +PREFERENCE because its executed shape usually matches the drawn one; a dynamic graph is +auditable too when it keeps that record within those caps. What is never evidence is a +design DIAGRAM on its own — *"here is what I planned"* is not *"here is what happened"*, +in either mode. The six-row table, the rest of the model — the fake-edge test, the diamond, the checker node before a convergence — and what a host actually executes when it fans out are one @@ -145,8 +152,12 @@ The long version is `references/audit.md`. The shape: Monday. This is the same rule `agent-evals` applies to eval rubrics and `seo-aeo-audit` to sites. -**The finding that ends most audits early:** the system has no evals. Everything downstream -is then unfalsifiable — including this audit. Say so first, and make it the first item. +**The finding most audits surface first:** the system has no evals. That is a finding +about UNKNOWN RELIABILITY — every *behavioural estimate* downstream is unfalsifiable, +including this audit's. It does NOT dissolve what is provable at the source: a +demonstrable double charge, a hardcoded secret, a deterministic race keep their own +findings and their own priority, set by the concrete harm — a general "no evals" never +masks a specific proven harm. --- @@ -189,7 +200,7 @@ prompt. ## Checklist — a harness worth shipping - [ ] Workflow-versus-agent decided deliberately, and the simpler option was actually tried -- [ ] Static-versus-dynamic decided too, and a run that must be auditable is static +- [ ] Static-versus-dynamic decided too — static preferred for predictability; a run that must be auditable keeps a complete execution record (not merely a static shape) - [ ] System prompt at the **right altitude** — heuristics, not hardcoded branches, not vague hope - [ ] Every status, category and enum the agent must produce is **enumerated in the prompt** - [ ] Today's date, and any other volatile context, injected rather than assumed diff --git a/plugins/agent-stack/skills/agent-harness/references/audit.md b/plugins/agent-stack/skills/agent-harness/references/audit.md index 7cd6ddc..3df42e8 100644 --- a/plugins/agent-stack/skills/agent-harness/references/audit.md +++ b/plugins/agent-stack/skills/agent-harness/references/audit.md @@ -88,10 +88,21 @@ Walk them in order. Later tracks assume earlier ones. - Is tool output treated as **untrusted input**? - Can an audit row prove a control was applied — does it carry the **policy version**? - Is there a deterministic limit anywhere consequential, or only probabilistic content checks? +- **The lethal trifecta is a specific EXFILTRATION pattern, not a full threat + model.** Private data + untrusted content + external comms in one session is + the exfiltration triangle — but a session MISSING one leg is not thereby + "safe". Audit **capabilities and effects SEPARATELY**: untrusted content + + a write capability, with no private data at all, is an unrelated destructive + effect (injected content corrupts state or takes a damaging action) and is + its OWN finding. "Only two of the three, therefore a PASS" is the mistake — + removing a trifecta leg removes THAT exfiltration path, not every risk. ### 7 — Evidence -- **Are there evals?** If not, this is finding number one and everything else is unfalsifiable. +- **Are there evals?** If not, that is finding number one — about unknown reliability: every + *behavioural estimate* in this audit is then unfalsifiable. Findings proven at the source + (an invariant read off the code, a deterministic reproduction) stand on their own and are + prioritized by their concrete harm, not discounted under the general finding. - Do they judge the **trajectory**, or only the final answer? - Has any production failure become a permanent fixture? - Is a judge calibrated against human labels, or trusted because it is a judge? @@ -110,6 +121,13 @@ Every finding carries one, and the tier is part of the finding: **Never present judgement as measured.** A finding whose tier is honest survives the meeting where it is challenged; one that is inflated loses the whole report. +Orthogonal to the tier, name the PROOF CLASS, because it decides what "no evals" does to +the finding: a **source-level invariant proof** (the race, the hardcoded secret, the +miswired timeout — read off the code) and a **deterministic reproduction** (a script that +shows the double charge every run) survive a system with no evals untouched; only a +**behavioural estimate** ("the agent usually recovers") needs an eval suite to be +falsifiable — and inherits the no-evals finding until one exists. + ## Priority — four axes, and no scalar `P = blast × confidence / effort` used to sit here, and it contradicted the two sections @@ -172,6 +190,10 @@ a finding whose tier is `judgement` says so there rather than being quietly disc - **Grading instead of planning.** A score ends the conversation the audit was meant to start — including a score assembled from honest axes. Publish the axes; do not multiply them. -- **Confusing "no evals" with "not measured yet."** It is the root finding; put it first, - because every other conclusion inherits it. +- **Confusing "no evals" with "not measured yet."** It is the root finding for every + behavioural estimate, which inherits it. A source-level proof or deterministic + reproduction does NOT inherit it — burying a demonstrable double charge under a general + "everything is unfalsifiable" is how the one finding with a victim gets deprioritized. +- **Treating a broken unit invariant with a prompt change.** Rule zero orders the + diagnosis; it does not convert a code bug into a wording bug. - **Reading a silent scanner as a clean system.** It is silent about what it can see. diff --git a/plugins/agent-stack/skills/agent-harness/references/tools.md b/plugins/agent-stack/skills/agent-harness/references/tools.md index 1f30519..06e9fb6 100644 --- a/plugins/agent-stack/skills/agent-harness/references/tools.md +++ b/plugins/agent-stack/skills/agent-harness/references/tools.md @@ -124,7 +124,12 @@ the mistake cannot be made**, rather than documenting the mistake. - An `enum` instead of a free-text field with a list of valid values in the description. - One tool that does the two-step correctly instead of two tools that must be ordered. - A required `confirm: true` on a destructive action, so a partially-formed call fails - closed. + closed. **But `confirm: true` is a SYNTAX GUARD, not user approval** — the MODEL can + set the boolean itself, so it proves only that the call is complete, never that a human + agreed. Real user approval is a **verifiable grant from a trusted control plane, bound to + the principal, action, exact arguments and an expiry** (or an already-existing user + authorization); a stale grant does NOT authorize changed arguments, and a client-supplied + boolean creates no authorization at all. ## Annotations, and the risk one tool cannot show you @@ -154,8 +159,12 @@ Three capabilities that are individually ordinary and jointly an exfiltration pa 2. exposure to **untrusted content**, 3. the ability to **communicate externally**. -Any two are safe. All three in one session mean untrusted content can instruct the agent to -read private data and send it out, and no prompt-level instruction reliably prevents it. +All three in one session mean untrusted content can instruct the agent to read private data +and send it out, and no prompt-level instruction reliably prevents it. But the trifecta +names a SUFFICIENT configuration for one SPECIFIC risk — private-data EXFILTRATION — not a +complete security model: **"any two are safe" over-claims.** Untrusted content plus a write +capability, with no access to private data at all, still lets injected content corrupt state +or take a damaging action; drop any leg and you have removed THAT triangle, not every risk. **The reason it belongs here rather than in a permission check:** the trifecta is a property of *the tool set assembled in a session*, so **per-tool analysis cannot see it**. Every tool diff --git a/plugins/agent-stack/skills/agent-interop/SKILL.md b/plugins/agent-stack/skills/agent-interop/SKILL.md index 436c484..b8bbe69 100644 --- a/plugins/agent-stack/skills/agent-interop/SKILL.md +++ b/plugins/agent-stack/skills/agent-interop/SKILL.md @@ -71,10 +71,29 @@ tasks, while MCP is more about agents using capabilities."* Real systems run bot server whose internals speak MCP — and that is the recommended architecture, not a compromise. -**The tell that you picked wrong:** if you find yourself inventing a task lifecycle, a -progress channel and a resumable handle on top of `tools/call`, you wanted A2A. If you find -yourself publishing an agent card for something that is one HTTP call with a JSON schema, -you wanted MCP. +**The dispatch criterion is WHAT the other side is, not how long it runs.** MCP = +a CAPABILITY / tool you control the shape of; A2A = an AUTONOMOUS PEER whose +outcome you delegate and whose insides you cannot see. **Duration is a SECOND +question, and it is about a Tasks CAPABILITY, not a protocol.** A long-running +FIXED job you own — a ten-minute export, a batch transform — is MCP with the +**Tasks** extension (a durable handle: poll, supply input mid-flight, retrieve +later; see `references/mcp.md`), NOT A2A. So decide by the routing set, and +never by the word *long-running*: + +- long-running FIXED export → **MCP Tasks, when the client/SDK negotiates that + extension**; +- autonomous outsourced negotiation → **A2A** (you delegate the outcome, not + the steps); +- **Tasks unsupported** by the reached client/SDK → an explicit fallback + (chunk the job, a job id the caller polls with a plain `tools/call`, or a + webhook) — reaching for A2A because Tasks is absent is picking a protocol to + dodge a missing extension. + +Check the client's ACTUALLY-negotiated extensions before building on Tasks; +inventing a task lifecycle on top of `tools/call` when Tasks IS available is +re-implementing the extension, and reaching for A2A when Tasks is merely +unsupported is the mis-route this audit closes. Publishing an agent card for +something that is one HTTP call with a JSON schema is still the MCP direction. --- diff --git a/plugins/agent-stack/skills/agent-interop/references/mcp-ship.md b/plugins/agent-stack/skills/agent-interop/references/mcp-ship.md index 851ca49..4f55a48 100644 --- a/plugins/agent-stack/skills/agent-interop/references/mcp-ship.md +++ b/plugins/agent-stack/skills/agent-interop/references/mcp-ship.md @@ -13,11 +13,47 @@ repeat it. ## Contents +- Every example names its SDK — distribution, import, tested version, lifecycle - Mounting into an existing web app - Auth middleware and a health endpoint - Client configuration - Debugging a client that will not connect +## Every example names its SDK — distribution, import, tested version, lifecycle + +`FastMCP` exists in TWO distributions, and they are not the same package: the +official `mcp` SDK (`from mcp.server.fastmcp import FastMCP`) and the +standalone `fastmcp` package, whose 2.x renamed the server class and moved the +transport args out of the constructor. A snippet that names only the class +names neither. Every executable example in this file is written against **the +official `mcp` SDK, pinned**: + +```text +# requirements.txt — the pin IS the example's identity +mcp==1.12.3 # the version these snippets were verified against (2026-08-13); +starlette==0.47.* # re-pin only together with a re-run of the acceptance below +uvicorn==0.35.* +``` + +```python +from mcp.server.fastmcp import FastMCP # official SDK — NOT `from fastmcp import FastMCP` +``` + +- **Current path (v2/current spec, pinned SDK above):** the snippets below, + as written. +- **v1 / standalone-`fastmcp` migration:** explicitly **out of scope** here. + Supporting it means its OWN fixture verified against its own pin — renaming + imports and hoping is how the constructor-args move ships a 404. +- **Lifecycle, in the verified order:** create the server → register tools → + build the ASGI app → register `/health` on the OUTER app → wrap with the + auth middleware → mount → serve. The health route registers BEFORE the auth + wrap, or the probe that says "up" needs a credential to say it. +- **Proof is a localhost protocol call, never a string in markdown.** The + acceptance boots the pinned example in a clean env and asserts: + `GET /health` → 200 · unauthenticated `/mcp` → 401 · then `initialize`, + `tools/list` and one `tools/call` succeed against `http://127.0.0.1`. A + snippet nobody booted is a hope with syntax highlighting. + ## Mounting into an existing web app The common production shape: you already run a FastAPI/Starlette app, and the MCP server diff --git a/plugins/agent-stack/skills/agent-interop/references/mcp.md b/plugins/agent-stack/skills/agent-interop/references/mcp.md index 1dbfbd9..b164783 100644 --- a/plugins/agent-stack/skills/agent-interop/references/mcp.md +++ b/plugins/agent-stack/skills/agent-interop/references/mcp.md @@ -219,7 +219,11 @@ Opt-in, negotiated, and worth checking before inventing an equivalent: - **Tasks** — a durable handle for long-running requests: poll for status, supply input mid-flight, retrieve the result later. This is the answer to "my tool takes ten minutes", - and it exists so you do not hold a connection open or invent a job table. + and it exists so you do not hold a connection open or invent a job table. **Duration is a + TASKS-capability question, never a reason to switch to A2A**: a long-running FIXED job you + own is MCP-with-Tasks. But Tasks is an OPT-IN, NEGOTIATED extension — check the client/SDK + actually supports it before building on it; where it is unsupported, the fallback is a + chunked job or a caller-polled job id over plain `tools/call`, not a protocol change. - **MCP Apps** — interactive UI rendered inline in the conversation. - **Skills over MCP** — structured instruction sets discovered and consumed through MCP, which is how a server ships Agent Skills rather than only tools. diff --git a/plugins/agent-stack/skills/agent-orchestrator/SKILL.md b/plugins/agent-stack/skills/agent-orchestrator/SKILL.md index 2cb9dde..d9d5fce 100644 --- a/plugins/agent-stack/skills/agent-orchestrator/SKILL.md +++ b/plugins/agent-stack/skills/agent-orchestrator/SKILL.md @@ -6,7 +6,7 @@ description: >- loops, pipelines with human checkpoints, provider routing with fallback/retry, memory architecture, retrieval and decay, context budgets, sub-agent coordination, error hierarchies; the work as a graph — parallel layers, fake edges, a checker before convergence; for resale: - tiered wallets, one markup boundary, two-phase commit across database and provider API, + tiered wallets, one markup boundary, the saga across database and provider API, spend-delta polling, budget and loop guards, per-tenant keys. Triggers - "agent system", "orchestrator", "tool calling", "sub-agent", "LLM router", "fallback chain", "human in the loop", "memory layer", "LLM billing", "token wallet", "агентная система", "оркестратор", @@ -282,10 +282,13 @@ comes from, and what this host actually executes are in Four rules, and these are the ones that change code: -- **Label every edge with what crosses it. No payload, no edge.** Run the fake-edge test - over any chain you inherited: write the steps as boxes, ask of each arrow whether data - from A actually enters B, and delete the arrows that only encode the order somebody - typed. Two or three per workflow is the normal yield. +- **Type every edge: data, control, authorization or resource.** Run the fake-edge + test over any chain you inherited — and delete an arrow only when it carries NONE + of the four: no payload, no causal constraint, no permission, no shared resource. + backup→migration, approval→charge and lease→edit carry no bytes and are real; + the arrows that only encode the order somebody typed are the two or three per + workflow the test normally yields. Before a fan-out, compare read/write sets — + read-only branches parallelise, two writers of one thing were a resource edge. - **`depends_on` is a claim, so execute by layer.** §5's executor walked `plan.stages` in list order beside a model that declared its dependencies — which serialises a plan that went to the trouble of saying it need not be. Kahn the graph; a cycle fails the plan @@ -294,9 +297,11 @@ Four rules, and these are the ones that change code: returns a hallucination, and the synthesis node cannot tell: it combines all three and answers confidently. The checker decides *usable / not usable* and nothing else, and the convergence depends on **the checker**, never directly on a branch. -- **Static unless you can name what forces dynamic.** A graph that picks its own next - nodes cannot be audited afterwards, because the shape that ran is not the shape anyone - drew. Where a run has to be explainable, that settles it. +- **Static unless you can name what forces dynamic** — for predictability, not + auditability. A run is auditable when it SAVES its execution record (nodes/edges/events + that ran, the policy version, deterministic budget/depth/node caps, provenance); a + dynamic graph that keeps that record is auditable too. A design diagram alone is never + evidence: the shape drawn is not the shape that ran. --- diff --git a/plugins/agent-stack/skills/agent-orchestrator/references/graph-engineering.md b/plugins/agent-stack/skills/agent-orchestrator/references/graph-engineering.md index 551c7a5..dd7b1a8 100644 --- a/plugins/agent-stack/skills/agent-orchestrator/references/graph-engineering.md +++ b/plugins/agent-stack/skills/agent-orchestrator/references/graph-engineering.md @@ -120,24 +120,43 @@ already have and ask which of its waits are real. ## 3. The fake-edge test -Five minutes, no tooling, and it is the highest-yield thing in this file. +Five minutes, no tooling, and it is the highest-yield thing in this file — +and its question is **typed**, because "does data cross?" alone deletes real +constraints (AS-04). An edge is one of four kinds: + +| Kind | What crosses | Example that MUST survive | +|---|---|---| +| **data** | A's output enters B | findings → draft | +| **control** | ordering only — a causal constraint with no bytes | backup → migration | +| **authorization** | a decision that permits B | approval → charge | +| **resource** | A and B touch one thing that tolerates one writer | two writes to one ledger | 1. Write every step as a box. 2. Draw an arrow between each pair of consecutive steps. -3. For each arrow ask: **does data from A actually enter B?** — not *"does B come after - A"*. -4. Yes → keep it, and **write the payload on the arrow**. -5. No → delete it. That wait was free to give away and you were paying for it. -6. Everything with no incoming arrow starts immediately. +3. For each arrow ask, in order: does data from A enter B? does B's SAFETY + depend on A having finished (control)? does A PERMIT B (authorization)? + do A and B contend for one resource? +4. Any yes → keep it, **type it, and write the rationale on the arrow** — + the payload for a data edge, the constraint for the other three. +5. No to all four → delete it. That wait encoded the order somebody typed. +6. Everything with no incoming arrow starts immediately — and **before any + fan-out, compare the branches' side-effect footprints and read/write + sets**: two read-only reviews genuinely parallelise; two writers of one + file were a resource edge nobody drew. 7. Everything with no outgoing arrow is a final output. -The tell that the test is being done honestly is step 4: if the payload cell is empty, -the edge is fake, and the person drawing it now has to say so out loud rather than -leaving the arrow in place because it looked orderly. +The tell that the test is being done honestly is step 4: an arrow with no +type and no rationale is fake, and the person drawing it now has to say so +out loud rather than leaving it in place because it looked orderly. The +inverse tell is step 5 done lazily: backup→migration, approval→charge and +lease→edit all carry NO payload, and deleting them for that is how a +migration runs against nothing — an empty payload cell justifies deletion +only when there is also no causal, permissive or resource constraint. -**Expect two or three fake edges in any workflow you have not run this against.** The -classic is *"review file A, then review file B"*: it reads as a sequence, and the review -of B never once looks at what A returned. +**Expect two or three fake edges in any workflow you have not run this +against.** The classic is *"review file A, then review file B"*: it reads as +a sequence, the review of B never looks at what A returned, and both are +read-only — no data, no control, no authorization, no resource. ## 4. The diamond @@ -263,13 +282,20 @@ grows: a node finishes, looks at what it found, and decides what should come nex | **static** | **always first** — switch only after the static version hits a wall you can name | | dynamic | the scope of the work depends on what is discovered along the way | | dynamic | a node must choose its successors from its own output | -| **never dynamic** | **you will need to audit exactly what ran and why** | - -The last row is a hard rule in this pack, not a preference. A dynamic graph's executed -shape is not the shape anybody drew, so *"here is the graph"* and *"here is what -happened"* stop being the same document — and every claim about the run becomes -unfalsifiable from the outside. That is the same failure `agent-evals` names when a -system has no durable trace. +| prefer static | **you will need to audit exactly what ran and why** — but see below | + +The last row is a PREFERENCE, not a hard ban — and the earlier draft got this wrong by +equating auditability with static structure, which conflates the plan drawn beforehand +with the execution graph saved afterward. **Auditability is a property of the RECORD, not +of the shape:** a run is auditable when its execution record is complete (every node, +edge and event that ran), the policy version is captured, and the run stayed inside +deterministic bounds — a **budget, a depth cap and a node cap**, each with provenance. A +static graph is preferred because its executed shape usually matches the drawn one and it +is predictable; a DYNAMIC graph under those caps, saving that record, passes the same +reconstruction audit — the SAME reconstruction that detects a deleted event or edge in +either mode. What never passes, static or dynamic, is a design DIAGRAM on its own: +*"here is the graph"* is not *"here is what happened"*. That is the same failure +`agent-evals` names when a system has no durable trace. **Most workflows that feel like they need a dynamic graph need a better static one.** Dynamic is more powerful and much harder to control; it is the second reach, never the @@ -364,7 +390,12 @@ session: ## Workflow defaults - A node with no declared dependency starts immediately; do not serialise by habit. -- Every declared dependency names the data it carries. No payload named ⇒ delete the edge. +- Every declared dependency carries a type (data/control/authorization/resource) + and a rationale. Delete an edge only when it has none of the four — an empty + payload alone never justifies deletion: backup→migration carries no bytes and + is real. +- Before a fan-out, compare the branches' side-effect footprints and read/write + sets; writers of one resource serialise, read-only branches run in parallel. - A checker sits between any parallel layer and the node that consumes it, and the consumer depends on the checker rather than on the layer. - A checker flags; it never silently passes an incomplete output. diff --git a/plugins/agent-stack/skills/agent-orchestrator/references/llm-proxy-billing.md b/plugins/agent-stack/skills/agent-orchestrator/references/llm-proxy-billing.md index 7aa297c..9f3c0c1 100644 --- a/plugins/agent-stack/skills/agent-orchestrator/references/llm-proxy-billing.md +++ b/plugins/agent-stack/skills/agent-orchestrator/references/llm-proxy-billing.md @@ -1,7 +1,7 @@ # Reselling LLM access — metering, wallets and guardrails **Load this when** the product resells LLM access: tiered wallets and the single -boundary where markup applies, two-phase commit across a database and a provider API +boundary where markup applies, the saga across a database and a provider API with compensating transactions, advisory locking, optimistic concurrency for reclaims, spend-delta polling and its three cases, budget / loop / auto-pause guardrails, per-tenant key lifecycle and healing, the refund waterfall, and model-routing @@ -18,7 +18,8 @@ the patterns hold for any upstream that issues per-tenant keys with limits. ## Contents - [The tiered wallet](#the-tiered-wallet) -- [Two-phase commit across a DB and an external API](#two-phase-commit-across-a-db-and-an-external-api) +- [The saga across a DB and an external API](#the-saga-across-a-db-and-an-external-api) +- [Reconciling an unknown](#reconciling-an-unknown) - [Serializing concurrent transfers](#serializing-concurrent-transfers) - [Optimistic concurrency for reclaims](#optimistic-concurrency-for-reclaims) - [Discovering spend you do not control](#discovering-spend-you-do-not-control) @@ -67,29 +68,76 @@ the constant changes. --- -## Two-phase commit across a DB and an external API +## The saga across a DB and an external API -You have a database you can roll back and an HTTP API you cannot. Order matters, -and so does what you do when step 2 fails. +You have a database you can roll back and an HTTP API you cannot. That pair is +a **saga** — local transactions stitched together by compensations — and it is +not two-phase commit: 2PC needs a coordinator both participants obey, and the +provider's API never agreed to prepare/commit. Naming it 2PC is how the next +defect ships, because 2PC has no *unknown* outcome, and an HTTP call to a +system you do not control has one all the time. -**DB first, API second, compensate on failure:** +Every operation that touches the provider carries an **`operation_id`**, minted +inside the DB transaction, and a state that moves +`pending → applied | unknown | compensated`: 1. Acquire the lock (below). 2. Read fresh balances **inside** the transaction — not before it. 3. Compute the transfer and apply the markup once. -4. Zero the source tier, increment the destination, write an audit row. +4. Zero the source tier, increment the destination, write the intent row — + `operation_id`, state `pending` — an outbox entry, not a log line. 5. Commit. -6. Call the provider to raise the key limit. -7. **On API failure: a compensating transaction restores every DB value and - writes a `compensation` audit row.** +6. Call the provider to raise the key limit, idempotently where the API allows + (send the `operation_id` as the idempotency key). +7. **On an outcome that proves the call did not apply** — a 4xx validation + refusal, a "no such key" — a compensating transaction restores every DB + value, writes a `compensation` audit row, and marks the operation + `compensated`. +8. **On an AMBIGUOUS outcome — a timeout, a connection reset after send, a + 5xx — the operation is marked `unknown` and is NOT compensated.** The + provider may have applied the change: compensating on a guess restores a + ledger the key no longer matches, and the money drifts in the direction you + cannot see. `unknown` resolves only by **reconciliation** — read the + provider's actual state (the key's real limit), then mark `applied` or + compensate on evidence. Until it resolves, the operation blocks retries of + itself: a retry of an `unknown` is how one top-up applies twice. The alternative — API first, DB second — leaves money on the key that your ledger does not know about, and no amount of retrying finds it again. The -compensating transaction is not optional politeness; it is the only thing that -makes step 6 recoverable. - -Log both the intent and the compensation. An audit trail that records only -successes cannot answer "where did the $35 go" six weeks later. +compensating transaction makes the *known* failure recoverable; the `unknown` +state is what keeps the ambiguous one honest. + +Log the intent, the outcome and the compensation, keyed by `operation_id`. An +audit trail that records only successes cannot answer "where did the $35 go" +six weeks later — and one that cannot say "we do not know yet" answers it +wrongly. + +## Reconciling an unknown + +Three rules, and every one exists because a late HTTP response is a message +from the past: + +- **Ask by the operation's own idempotency key.** Reconciliation queries the + provider for what happened to THIS `operation_id` — never "read the limit + and guess whose change it reflects". Ambient state is the sum of every + operation that ever landed; only the key isolates yours. +- **The tenant's ledger carries a revision, and every resolve is a CAS.** A + reconcile or compensation writes only if the revision it read is still + current; a late or concurrent response that lost the race aborts and + re-reads, it never blind-writes. Without this, the response to operation A — + arriving after operation B moved the same tenant's ledger — "restores" + values B already superseded, and the compensation itself becomes the + corruption. +- **Compensate only your own confirmed operation.** A compensation names its + `operation_id`, reverses exactly that operation's delta, and runs only after + reconciliation confirmed THAT operation did not apply. A response for A is + never grounds to touch B's rows — however tempting the arithmetic looks. + +**Repeated reconciliation is idempotent.** `unknown → applied` and +`unknown → compensated` are one-way edges: resolving an already-resolved +operation reads its state and stops — zero new writes, zero new audit rows. A +reconciler that runs twice (and it will: cron plus a manual "Sync now" is the +normal case, not the weird one) must find nothing left to do the second time. --- @@ -141,16 +189,30 @@ discover spend by **polling a cumulative counter and taking the delta**: delta = currentUsage - lastRecordedUsage ``` -Three cases, and only the first is obvious: - -- `lastRecordedUsage == 0 && currentUsage > 0` → **seed the baseline, record - nothing.** Recording it charges the tenant for everything spent before you - started watching. -- `currentUsage > lastRecordedUsage` → record `delta`, then immediately enforce - budgets (below). -- `currentUsage < lastRecordedUsage` → the key was recreated. **Resync the - baseline, record nothing.** A negative delta treated as spend credits money - that was never returned. +**Zero is a value, not an absence.** The baseline row carries three fields +BESIDE the sum — `baseline_initialized`, `observed_at`, and +`provider_key_generation` (the key's id or created-at, whatever the provider +lets you read) — because `lastRecordedUsage == 0` has two meanings that cost +money to conflate: "never watched" and "watched from zero". Testing the sum +for zero eats the first REAL spend of every key you watched from birth, +silently, as "seeding". + +Four cases, decided by the flags, never by the sum: + +- `!baseline_initialized` → **seed the baseline, record nothing**, set + `baseline_initialized`, stamp `observed_at` and the generation. Recording + here charges the tenant for everything spent before you started watching. +- initialized, `currentUsage > lastRecordedUsage` → record `delta` — including + the very first delta of a key whose baseline is a genuine 0 — then + immediately enforce budgets (below). +- initialized, `currentUsage < lastRecordedUsage`, **generation changed** → + the key really was recreated: resync the baseline to the new generation, + record nothing. The new key's next increase is recorded normally. +- initialized, `currentUsage < lastRecordedUsage`, **same generation** → + **ANOMALY.** Do not resync, do not record, do not guess "recreated" — a + counter that went backwards on the same key is the provider disagreeing + with your ledger, and reconciliation (above) owns it. A guessed resync here + quietly forgives the difference forever. Sync your stored limit from the provider's authoritative value on the same pass — under the lock, with a re-read, so the sync does not clobber a transfer that diff --git a/plugins/agent-stack/skills/agent-orchestrator/references/memory-architecture.md b/plugins/agent-stack/skills/agent-orchestrator/references/memory-architecture.md index 1e7abb5..f3b139b 100644 --- a/plugins/agent-stack/skills/agent-orchestrator/references/memory-architecture.md +++ b/plugins/agent-stack/skills/agent-orchestrator/references/memory-architecture.md @@ -235,10 +235,12 @@ The progression, and both ends are wrong: access control**. **What this pack already has, and what it is not.** `agent-sync` gives leases, race-free id -reservation and a run journal: it decides *who may write this file right now*. That is -coordination, and it is not shared memory — it says nothing about what an agent should be -allowed to *read*, or whose experiential memory is trustworthy enough to act on. An agent -system that needs both needs both. +reservation and a run journal: it decides *who may write this file right now* — precisely, +via a per-file **resource claim**, because a task lease authorizes the task and not the file +(SY-04), and its guarantee is a real cross-machine compare-and-swap only under the git lease +backend, advisory otherwise. That is coordination, and it is not shared memory — it says +nothing about what an agent should be allowed to *read*, or whose experiential memory is +trustworthy enough to act on. An agent system that needs both needs both. **The design rule:** make shared writes **attributed and scoped**. An entry carries who wrote it and under what role, and a reader may weigh it accordingly. Unattributed shared diff --git a/plugins/agent-stack/skills/agent-orchestrator/references/memory-lifecycle.md b/plugins/agent-stack/skills/agent-orchestrator/references/memory-lifecycle.md index 255b43b..cc877ff 100644 --- a/plugins/agent-stack/skills/agent-orchestrator/references/memory-lifecycle.md +++ b/plugins/agent-stack/skills/agent-orchestrator/references/memory-lifecycle.md @@ -18,7 +18,8 @@ long-term behaviour is a consequence of those frequencies, not of separate boxes - [3. Evolution — consolidation](#3-evolution--consolidation) - [4. Evolution — updating, and the stability–plasticity dilemma](#4-evolution--updating-and-the-stabilityplasticity-dilemma) - [5. Evolution — forgetting](#5-evolution--forgetting) -- [6. What this pack already implements](#6-what-this-pack-already-implements) +- [6. The temporal evidence lifecycle](#6-the-temporal-evidence-lifecycle) +- [7. What this pack already implements](#7-what-this-pack-already-implements) ## 1. Formation — five ways to turn experience into an entry @@ -134,7 +135,38 @@ constraint, many memory systems avoid directly deleting certain memories."* Deletion remains a **correctness and privacy** operation — a person asking to be forgotten is not a capacity decision, and `memory-architecture.md` §8 covers it. -## 6. What this pack already implements +## 6. The temporal evidence lifecycle + +One record's timeline, and the two events that are NOT on it: + +``` +born ──▶ corroborated ──▶ superseded ──▶ (restored) + │ (independent (reversible: un-supersede puts it back; + │ provenance is_active=False, nothing was deleted + │ only) superseded_by) + └──▶ expired (volatile fact past its freshness window — + out of default retrieval WHATEVER its confidence, + verified included) +``` + +- **Retrieval is not a lifecycle event.** Reading a record — or the agent + restating it in its own words — moves nothing: not confidence, not + freshness, not activity. A self-generated repeat scored as confirmation is + the compounding error `patterns.md` → Confidence Management refuses; only + evidence with independent provenance corroborates. +- **Supersession is an annotation, and it is reversible.** A new dated fact + sets `is_active=False` + `superseded_by` on the old one and KEEPS it: the + chain is walkable from either end, an explicit query still reaches the old + value, and restoring it (the correction turned out wrong) is clearing two + fields, not resurrecting a deleted row. The dated history is the audit + trail of what the system believed when. +- **`verified` is about confidence, never about time.** A volatile fact — + a quota, a price, a rate limit — carries a freshness window from its + `validity` field, and past that window it leaves default retrieval even at + confidence 1.0. Verification exempts a record from confidence DECAY; + nothing exempts a fact about the present from the present. + +## 7. What this pack already implements Stated so this file is read as an extension and not as a replacement: @@ -146,9 +178,9 @@ Stated so this file is read as an extension and not as a replacement: | Forgetting, time-based | `patterns.md` → Confidence Management | | Global integration, cross-scope | `patterns.md` → Cross-Resource Learning Transfer | | **Frequency-based forgetting** | **nowhere — and the long-tail trap above is why that is a deliberate omission rather than a gap to close carelessly** | -| **Temporal annotation instead of deletion** | **nowhere** — Conflict Resolution currently resolves rather than annotates | +| **Temporal annotation instead of deletion** | `patterns.md` → Conflict Resolution — supersession sets `is_active=False` + `superseded_by` and the old record stays in history, reversible and reachable by explicit query | | **Dual-phase updating** | **nowhere** — the pack updates inline | -The last three are named as absent rather than quietly added: each is a real change to a +The remaining two are named as absent rather than quietly added: each is a real change to a mechanism that is in production, and this file's job is to say what the options are, not to change `patterns.md` from a survey. diff --git a/plugins/agent-stack/skills/agent-orchestrator/references/patterns.md b/plugins/agent-stack/skills/agent-orchestrator/references/patterns.md index c6249bd..20c5ee4 100644 --- a/plugins/agent-stack/skills/agent-orchestrator/references/patterns.md +++ b/plugins/agent-stack/skills/agent-orchestrator/references/patterns.md @@ -364,26 +364,36 @@ class LearningAnalyzer: ## Confidence Management +Confidence measures **corroboration, not match frequency**: it moves only on +evidence with independent provenance — another session, another agent, an +observed outcome. Retrieving a record, or the agent restating it in its own +words, bumps nothing: a self-generated repeat scored as confirmation is how an +early mistake compounds into a "high-confidence" one. + ``` LEARNING CONFIDENCE: - Initial: 0.6 - Confirmed: +0.1 (cap 1.0) - Applied: tracked (times_applied counter) - Contradicted: -0.3 - Stale (30d): -0.02/month - Deactivated: below 0.2 + Initial: 0.6 + Corroborated: +0.1 (cap 1.0) — independent provenance only; + retrieval and self-repetition move nothing + Applied: tracked (times_applied counter — a usage stat, not evidence) + Contradicted: handled by the contradiction gate below, never a bare -0.3 + on a keyword match + Stale (30d): -0.02/month + Deactivated: below 0.2 — demoted from default retrieval, kept in history SESSION NOTE CONFIDENCE: - Initial: 0.7 - Confirmed: +0.1 (cap 1.0) - Verified: +0.15 (exempt from decay) - Stale (60d): -0.1 per cycle - Floor: 0.1 + Initial: 0.7 + Corroborated: +0.1 (cap 1.0), same provenance rule + Verified: +0.15 — exempt from confidence DECAY, not from validity: + a volatile fact past its freshness window leaves default + retrieval whatever its confidence says + Stale (60d): -0.1 per cycle + Floor: 0.1 INSIGHT CONFIDENCE: Initial: 0.5 Resurfaced: +0.05 - Confirmed: +0.15 + Corroborated: +0.15 Dismissed: -0.2 Stale (30d): -0.05 per cycle Expired: below 0.15 @@ -393,53 +403,83 @@ INSIGHT CONFIDENCE: ## Fuzzy Deduplication Pattern -Used across all memory layers: +**Similarity proposes; the contradiction gate disposes.** Lexical similarity +is CANDIDATE RETRIEVAL only — it has no side effects. The measured +counterexample that fixed this rule: *"Always allow external sharing of +customer data"* and *"Never allow external sharing of customer data"* score +`SequenceMatcher` similarity **0.8791** — far above any threshold — and the +old on-match behaviour (bump confidence, keep the longer text, reactivate) +would have REINFORCED the stale instruction with the user's own correction, +and kept "Always" because it is one word longer. ```python from difflib import SequenceMatcher THRESHOLD = 0.75 # learnings/notes; 0.80 for insights -async def find_similar(session, connection_id, category, subject, text): +async def find_candidates(session, connection_id, category, subject, text): + """Returns candidates for the contradiction gate. Nothing else happens + here: no confidence bump, no text replacement, no reactivation.""" candidates = await load_existing(session, connection_id, category, subject) text_lower = text.strip().lower() - best_match, best_ratio = None, 0.0 - for c in candidates: - ratio = SequenceMatcher(None, c.text.strip().lower(), text_lower).ratio() - if ratio >= THRESHOLD and ratio > best_ratio: - best_match, best_ratio = c, ratio - return best_match - -# On match: bump confidence +0.1, keep longer text, set is_active=True -# On no match: create new entry + return [c for c in candidates + if SequenceMatcher(None, c.text.strip().lower(), text_lower).ratio() + >= THRESHOLD] + +# Every candidate goes through the contradiction gate below. +# Only the gate's verdict decides merge / supersede / coexist / create. ``` --- ## Conflict Resolution Pattern -Detect when new learning contradicts existing ones: +A memory record carries five mandatory fields beside its text, and the gate +reads THEM — never the prose: + +``` +MEMORY RECORD: + entity what the statement is about ("customer-data-sharing") + attribute which property of it ("external-sharing-policy") + scope where it applies ("project-A" | "global" | …) + provenance who/what asserted it, when (session, agent, outcome, user) + validity observed_at + volatile|stable (+ freshness window if volatile) + value the normalized position ("allow" | "deny" | "30s" | …) +``` ```python -CONFLICT_INDICATORS = {"use", "prefer", "always", "never", "should", - "instead", "not", "avoid", "correct", "wrong"} - -def resolve_conflicts(existing_learnings, new_lesson, new_confidence): - new_keywords = {w for w in new_lesson.lower().split() if w in CONFLICT_INDICATORS} - for old in existing_learnings: - old_keywords = {w for w in old.lesson.lower().split() if w in CONFLICT_INDICATORS} - shared = new_keywords & old_keywords - if not shared: continue - - has_negation_flip = ( - ("not" in new_keywords) != ("not" in old_keywords) or - ("never" in new_keywords) != ("never" in old_keywords) or - ("avoid" in new_keywords) != ("avoid" in old_keywords)) - - if has_negation_flip and old.confidence <= new_confidence: - old.is_active = False # superseded +def contradiction_gate(old, new): + if (old.entity, old.attribute) != (new.entity, new.attribute): + return "unrelated" # similarity alone never merges anything + if not scopes_overlap(old.scope, new.scope): + return "coexist" # a correction wins only in its own scope + if values_compatible(old.value, new.value): + return "corroborates" # +confidence iff provenance is independent + return "contradicts" + +def apply_verdict(verdict, old, new): + if verdict == "contradicts": + # Temporal supersession, reversible: the old record STAYS in history. + old.is_active = False + old.superseded_by = new.id # never deleted, never bumped + return create(new) # starts at its own initial confidence + if verdict == "corroborates" and independent(old.provenance, new.provenance): + old.confidence = min(1.0, old.confidence + 0.1) + old.provenance.append(new.provenance) + return old + if verdict in ("coexist", "unrelated"): + return create(new) # both live; different scope or subject + return old # self-repetition: no change at all ``` +Keyword heuristics (negation flips, `always`/`never` pairs) may FLAG a pair +for the gate; they never decide it. The second measured counterexample is +why: *"Use Python"* and *"Never use production credentials"* share `use` and +a negation flip, and the old keyword rule could supersede one with the other +— two statements about different entities entirely. A number or unit change +("timeout is 30s" → "timeout is 60s") is a contradiction the negation +heuristic cannot see and the value comparison catches. + --- ## Cross-Resource Learning Transfer diff --git a/test/audit_regressions/fix-as-01.01.py b/test/audit_regressions/fix-as-01.01.py new file mode 100644 index 0000000..2a52b48 --- /dev/null +++ b/test/audit_regressions/fix-as-01.01.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +"""FIX-AS-01.01 — saga state model (sherlock audit, finding AS-01). + +The finding: the billing doctrine called its DB+HTTP pattern "two-phase +commit" and prescribed compensating an API failure unconditionally — but an +HTTP outcome can be AMBIGUOUS (timeout, reset after send, 5xx), and 2PC has no +vocabulary for that. Compensating on a guess restores a ledger the provider's +key no longer matches, in the direction nobody can see. + +The fix under test: the doctrine names the pattern a saga, every provider +operation carries an operation_id and a state pending → applied | unknown | +compensated, a documented HTTP timeout lands in `unknown` — never +auto-compensated — and resolves only by reconciliation on evidence. + +The state model is also run as behaviour: a transition table implementing +exactly the documented states, driven over the acceptance cases both ways. + +Standard library only. +""" +import os +import re +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", + "references", "llm-proxy-billing.md") +SKILL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", "SKILL.md") + +checks = 0 +failures = [] + + +def case(name, fn): + global checks + try: + fn() + checks += 1 + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +# ------------------------------------------------------------------ the doctrine + + +def t_doctrine_names_the_saga_not_2pc(): + text = open(DOC, encoding="utf-8").read() + assert "## The saga across a DB and an external API" in text, "the section lost its saga name" + assert "not two-phase commit" in text, "the doctrine no longer says the pattern is not 2PC" + for needle in ("operation_id", "pending → applied | unknown | compensated", + "marked `unknown` and is NOT compensated", + "resolves only by **reconciliation**", + "outbox entry, not a log line", + "idempotency key"): + assert needle in text, f"the doctrine no longer states {needle!r}" + # The old prescription must be gone: no unconditional compensate-on-API-failure. + assert "On API failure: a compensating transaction" not in text, \ + "the unconditional compensation prescription survived the rewrite" + # The term is corrected everywhere in the file, headings and prose alike. + stale = [l for l in text.splitlines() if re.search(r"two-phase commit", l, re.I) + and "not two-phase commit" not in l] + assert not stale, f"the file still calls it 2PC: {stale[:2]}" + skill = open(SKILL, encoding="utf-8").read() + assert "two-phase commit" not in skill, "SKILL.md still advertises the pattern as 2PC" + assert "saga across database and provider API" in skill, \ + "SKILL.md description lost the saga wording" + + +# ------------------------------- the documented state model, run as behaviour + + +STATES = {"pending", "applied", "unknown", "compensated"} +# outcome of the provider call → what the doctrine says happens +DEFINITE_FAILURE = "definite_failure" # 4xx refusal: provably not applied +AMBIGUOUS = "ambiguous" # timeout / reset after send / 5xx +SUCCESS = "success" + + +class Saga: + def __init__(self): + self.state = "pending" + self.compensations = 0 + self.audit = ["intent"] + + def outcome(self, kind): + assert self.state == "pending", f"outcome on a {self.state} operation" + if kind == SUCCESS: + self.state = "applied" + elif kind == DEFINITE_FAILURE: + self.compensations += 1 + self.audit.append("compensation") + self.state = "compensated" + elif kind == AMBIGUOUS: + self.state = "unknown" # and nothing else: no compensation, no retry + self.audit.append("unknown") + return self.state + + def reconcile(self, provider_applied): + assert self.state == "unknown", "reconciliation is for unknown outcomes" + if provider_applied: + self.state = "applied" + else: + self.compensations += 1 + self.audit.append("compensation") + self.state = "compensated" + return self.state + + def may_retry(self): + """A retry of an unknown is how one top-up applies twice.""" + return self.state == "compensated" + + +def t_timeout_is_not_auto_compensated(): + s = Saga() + assert s.outcome(AMBIGUOUS) == "unknown" + assert s.compensations == 0, "a timeout was compensated on a guess" + assert not s.may_retry(), "an unknown operation was offered for retry" + + +def t_definite_failure_compensates_once(): + s = Saga() + assert s.outcome(DEFINITE_FAILURE) == "compensated" + assert s.compensations == 1 + assert "compensation" in s.audit + assert s.may_retry(), "a compensated operation must be retryable" + + +def t_unknown_resolves_only_by_evidence(): + applied = Saga() + applied.outcome(AMBIGUOUS) + assert applied.reconcile(provider_applied=True) == "applied" + assert applied.compensations == 0, \ + "the provider had applied the change and the ledger was rolled back anyway" + + not_applied = Saga() + not_applied.outcome(AMBIGUOUS) + assert not_applied.reconcile(provider_applied=False) == "compensated" + assert not_applied.compensations == 1 + + +def t_success_path_stays_clean(): + s = Saga() + assert s.outcome(SUCCESS) == "applied" + assert s.compensations == 0 + assert s.audit == ["intent"] + + +def main(): + case("the doctrine names the saga, the states, and never 2PC", + t_doctrine_names_the_saga_not_2pc) + case("a documented HTTP timeout is unknown, never auto-compensated", + t_timeout_is_not_auto_compensated) + case("a definite failure compensates exactly once, with its audit row", + t_definite_failure_compensates_once) + case("unknown resolves only by reconciliation evidence", t_unknown_resolves_only_by_evidence) + case("the success path stays clean", t_success_path_stays_clean) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print(f"OK ({checks} checks)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-01.02.py b/test/audit_regressions/fix-as-01.02.py new file mode 100644 index 0000000..151dfae --- /dev/null +++ b/test/audit_regressions/fix-as-01.02.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""FIX-AS-01.02 — reconciliation and serialization (sherlock audit, AS-01, +second leaf; depends on FIX-AS-01.01's saga state model). + +The contract under test: an unknown outcome is reconciled BY the operation's +own idempotency key (never by guessing from ambient state); the tenant's +ledger carries a revision and every resolve is a CAS — a late or concurrent +HTTP response that lost the race aborts and re-reads, never blind-writes; a +compensation reverses only its OWN confirmed operation; and repeated +reconciliation is idempotent — one-way edges, zero new writes the second time. + +Acceptance, run as behaviour: a late response for operation A cannot +compensate over operation B's newer ledger state, and reconciling twice +changes nothing the second time. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", + "references", "llm-proxy-billing.md") + +checks = 0 +failures = [] + + +def case(name, fn): + global checks + try: + fn() + checks += 1 + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +# ------------------------------------------------------------------ the doctrine + + +def t_doctrine_states_the_three_rules(): + text = open(DOC, encoding="utf-8").read() + flat = " ".join(text.split()) + for needle in ("## Reconciling an unknown", + "Ask by the operation's own idempotency key", + "only the key isolates yours", + "every resolve is a CAS", + "aborts and re-reads, it never blind-writes", + "Compensate only your own confirmed operation", + "never grounds to touch B's rows", + "Repeated reconciliation is idempotent", + "one-way edges"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + assert "[Reconciling an unknown](#reconciling-an-unknown)" in text, \ + "the Contents list does not route to the section" + + +# --------------------- the documented rules, run as behaviour + + +class Ledger: + """A tenant ledger with a revision, exactly as the doctrine states it.""" + + def __init__(self, balance=100): + self.balance = balance + self.revision = 0 + self.audit = [] + + def read(self): + return {"balance": self.balance, "revision": self.revision} + + def cas_write(self, expected_revision, delta, note): + if expected_revision != self.revision: + return False # lost the race: abort and re-read, never blind-write + self.balance += delta + self.revision += 1 + self.audit.append(note) + return True + + +class Provider: + """The external API, queryable by idempotency key — never by guesswork.""" + + def __init__(self): + self.applied_ops = set() + + def apply(self, op_id): + self.applied_ops.add(op_id) + + def did_apply(self, op_id): + return op_id in self.applied_ops + + +class Op: + def __init__(self, op_id, delta): + self.op_id = op_id + self.delta = delta + self.state = "unknown" # this leaf starts where FIX-AS-01.01 left off + self.read_revision = None # what the late response saw + + def reconcile(self, ledger, provider): + """One-way edges; CAS on the revision read NOW; own delta only.""" + if self.state != "unknown": + return self.state # idempotent: nothing left to do + snapshot = ledger.read() + if provider.did_apply(self.op_id): + self.state = "applied" + return self.state + # confirmed not applied: compensate exactly this operation's delta + if not ledger.cas_write(snapshot["revision"], -self.delta, + f"compensation:{self.op_id}"): + return "retry" # the ledger moved: re-read, never blind-write + self.state = "compensated" + return self.state + + +def t_late_response_cannot_compensate_anothers_ledger(): + ledger, provider = Ledger(balance=100), Provider() + op_a = Op("op-a", delta=15) + ledger.cas_write(0, +15, "apply:op-a") # A applied locally, outcome unknown + + stale_revision = ledger.read()["revision"] # the late response's view of the world + op_b = Op("op-b", delta=30) + provider.apply("op-b") + ledger.cas_write(1, +30, "apply:op-b") # B lands and moves the revision + + # A's late compensation arrives holding the pre-B revision: a blind write here + # would "restore" values B already superseded. + ok = ledger.cas_write(stale_revision, -15, "compensation:op-a") + assert not ok, "a stale revision blind-wrote over a newer ledger" + assert ledger.balance == 145 and "compensation:op-a" not in ledger.audit + + # the honest path: re-read, then compensate ONLY op-a's delta + verdict = op_a.reconcile(ledger, provider) + assert verdict == "compensated" + assert ledger.balance == 130, f"the compensation touched more than op-a's delta: {ledger.balance}" + assert "apply:op-b" in ledger.audit and ledger.audit[-1] == "compensation:op-a" + + +def t_reconciliation_asks_by_key_not_by_guess(): + ledger, provider = Ledger(), Provider() + provider.apply("op-x") + op = Op("op-x", delta=10) + assert op.reconcile(ledger, provider) == "applied" + assert ledger.audit == [], "an applied operation was compensated anyway" + + other = Op("op-y", delta=10) # same delta, different key + assert other.reconcile(ledger, provider) == "compensated", \ + "the reconciler credited op-y with op-x's application — it guessed by value" + + +def t_repeated_reconciliation_is_idempotent(): + ledger, provider = Ledger(), Provider() + op = Op("op-z", delta=20) + first = op.reconcile(ledger, provider) + assert first == "compensated" + audit_after_first = list(ledger.audit) + revision_after_first = ledger.read()["revision"] + for _ in range(3): + assert op.reconcile(ledger, provider) == "compensated" + assert ledger.audit == audit_after_first, "a repeat reconcile wrote new audit rows" + assert ledger.read()["revision"] == revision_after_first, \ + "a repeat reconcile moved the revision" + + +def t_lost_cas_retries_and_lands_once(): + ledger, provider = Ledger(), Provider() + op = Op("op-r", delta=5) + # somebody moves the ledger between the reconciler's read and its write — + # simulated by bumping the revision after the snapshot would be taken + snapshot = ledger.read() + ledger.cas_write(snapshot["revision"], +1, "interloper") + assert not ledger.cas_write(snapshot["revision"], -5, "compensation:op-r"), \ + "the stale write went through" + assert op.reconcile(ledger, provider) == "compensated" # fresh read, lands once + assert ledger.audit.count("compensation:op-r") == 1 + + +def main(): + case("the doctrine states the three rules and the idempotence", + t_doctrine_states_the_three_rules) + case("a late response cannot compensate another's ledger", + t_late_response_cannot_compensate_anothers_ledger) + case("reconciliation asks by key, never by guess", t_reconciliation_asks_by_key_not_by_guess) + case("repeated reconciliation is idempotent", t_repeated_reconciliation_is_idempotent) + case("a lost CAS retries and lands exactly once", t_lost_cas_retries_and_lands_once) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print(f"OK ({checks} checks)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-02.01.py b/test/audit_regressions/fix-as-02.01.py new file mode 100644 index 0000000..489aa42 --- /dev/null +++ b/test/audit_regressions/fix-as-02.01.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""FIX-AS-02.01 — zero baseline is not an absent baseline (sherlock audit). + +The finding: the spend-discovery rule tested `lastRecordedUsage == 0` to mean +"never watched" — so a key genuinely watched from zero had its FIRST real +spend eaten as baseline seeding. And any decrease was guessed as "key +recreated" and silently resynced, forgiving real discrepancies forever. + +The fix under test: baseline_initialized / observed_at / +provider_key_generation live beside the sum; zero is a valid value; a +decrease on the SAME generation is an anomaly for reconciliation, never a +guessed resync. + +Acceptance sequences, each with its predetermined ledger record and state: +uninitialized→5 (seed, no record), initialized(0)→5 (record 5 — the first +real spend counts), 5→8 (record 3), 8→2 same generation (anomaly, no +record), generation change (resync; the new key's first spend counted). + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", + "references", "llm-proxy-billing.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_the_split(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("Zero is a value, not an absence", + "`baseline_initialized`, `observed_at`", + "provider_key_generation", + "decided by the flags, never by the sum", + "including the very first delta of a key whose baseline is a genuine 0", + "same generation** → **ANOMALY", + "A guessed resync here quietly forgives the difference forever"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + assert "lastRecordedUsage == 0 && currentUsage > 0" not in flat, \ + "the zero-means-absent test survived" + + +# ------------------- the documented state machine, run over the sequences + + +class Tracker: + def __init__(self): + self.initialized = False + self.baseline = 0 + self.generation = None + self.observed_at = None + self.ledger = [] + self.state = "ok" + + def poll(self, usage, generation, at): + if not self.initialized: + self.initialized = True + self.baseline, self.generation, self.observed_at = usage, generation, at + return ("seeded", None) + if generation != self.generation: + self.baseline, self.generation, self.observed_at = usage, generation, at + return ("resynced-new-generation", None) + if usage > self.baseline: + delta = usage - self.baseline + self.baseline, self.observed_at = usage, at + self.ledger.append(delta) + return ("recorded", delta) + if usage < self.baseline: + self.state = "anomaly" + return ("anomaly", None) + self.observed_at = at + return ("unchanged", None) + + +def t_acceptance_sequences(): + tr = Tracker() + assert tr.poll(5, "gen-1", 1) == ("seeded", None), "uninitialized→5 must seed" + assert tr.ledger == [], "seeding recorded spend" + + tr2 = Tracker() + tr2.poll(0, "gen-1", 1) # initialized at a GENUINE zero + verdict = tr2.poll(5, "gen-1", 2) + assert verdict == ("recorded", 5), \ + f"the first real spend of a watched-from-zero key was eaten: {verdict}" + assert tr2.ledger == [5] + + assert tr2.poll(8, "gen-1", 3) == ("recorded", 3) + assert tr2.ledger == [5, 3] + + verdict = tr2.poll(2, "gen-1", 4) # decrease, SAME generation + assert verdict == ("anomaly", None), f"a same-key decrease was explained away: {verdict}" + assert tr2.state == "anomaly" and tr2.ledger == [5, 3], \ + "the anomaly wrote or resynced anyway" + + tr3 = Tracker() + tr3.poll(8, "gen-1", 1) + assert tr3.poll(2, "gen-2", 2) == ("resynced-new-generation", None), \ + "a generation change was not resynced" + assert tr3.poll(6, "gen-2", 3) == ("recorded", 4), \ + "the new key's first spend was not counted" + assert tr3.ledger == [4] + + +def t_zero_then_zero_stays_clean(): + tr = Tracker() + tr.poll(0, "g", 1) + assert tr.poll(0, "g", 2) == ("unchanged", None) + assert tr.state == "ok" and tr.ledger == [] + + +def main(): + case("the doctrine splits zero from absent", t_doctrine_states_the_split) + case("all five acceptance sequences produce their predetermined records", + t_acceptance_sequences) + case("zero then zero is unchanged, not an event", t_zero_then_zero_stays_clean) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-03.01.py b/test/audit_regressions/fix-as-03.01.py new file mode 100644 index 0000000..658475c --- /dev/null +++ b/test/audit_regressions/fix-as-03.01.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""FIX-AS-03.01 — memory identity and contradiction (sherlock audit, AS-03). + +The finding: fuzzy dedup treated SequenceMatcher >= 0.75 as identity — so +"Always allow external sharing of customer data" and "Never allow external +sharing of customer data" (similarity 0.8791) merged, the confidence bump +REINFORCED the stale instruction with the user's own correction, and "keep +the longer text" kept "Always". The keyword conflict rule could also +supersede "Use Python" with "Never use production credentials". + +The fix under test, run as the documented behaviour: similarity is candidate +retrieval only; records carry entity/attribute/scope/provenance/validity; +the contradiction gate decides merge/supersede/coexist; a correction wins +only in its own scope; the old fact stays in history; self-repetition bumps +nothing; verified does not exempt a volatile fact from freshness. + +Standard library only. +""" +import os +import sys +from difflib import SequenceMatcher + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +REFS = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", + "references") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_the_gate(): + flat = " ".join(open(os.path.join(REFS, "patterns.md"), encoding="utf-8") + .read().split()) + for needle in ("Similarity proposes; the contradiction gate disposes", + "0.8791", + "no confidence bump, no text replacement, no reactivation", + "corroboration, not match frequency", + "a correction wins only in its own scope", + "never deleted, never bumped", + "exempt from confidence DECAY, not from validity", + "they never decide it"): + assert needle in flat, f"patterns.md no longer states {needle!r}" + for dead in ("bump confidence +0.1, keep longer text, set is_active=True", + "Verified: +0.15 (exempt from decay)"): + assert dead not in flat, f"the old behaviour survived: {dead!r}" + life = " ".join(open(os.path.join(REFS, "memory-lifecycle.md"), + encoding="utf-8").read().split()) + assert "supersession sets `is_active=False` + `superseded_by`" in life + assert "Conflict Resolution currently resolves rather than annotates" not in life + + +# ---------------- the documented record, gate and application, executed + + +class Rec: + _n = 0 + + def __init__(self, text, entity, attribute, scope, provenance, value, + volatile=False, fresh=True, confidence=0.6): + Rec._n += 1 + self.id = Rec._n + self.text, self.entity, self.attribute = text, entity, attribute + self.scope, self.value = scope, value + self.provenance = [provenance] + self.volatile, self.fresh = volatile, fresh + self.confidence = confidence + self.is_active, self.superseded_by = True, None + + +def find_candidates(store, text, threshold=0.75): + tl = text.strip().lower() + return [c for c in store + if SequenceMatcher(None, c.text.strip().lower(), tl).ratio() >= threshold] + + +def contradiction_gate(old, new): + if (old.entity, old.attribute) != (new.entity, new.attribute): + return "unrelated" + if old.scope != new.scope and "global" not in (old.scope, new.scope): + return "coexist" + if old.value == new.value: + return "corroborates" + return "contradicts" + + +def apply_verdict(verdict, old, new, store): + if verdict == "contradicts": + old.is_active = False + old.superseded_by = new.id + store.append(new) + return new + if verdict == "corroborates" and new.provenance[0] != old.provenance[0]: + old.confidence = min(1.0, old.confidence + 0.1) + old.provenance += new.provenance + return old + if verdict in ("coexist", "unrelated"): + store.append(new) + return new + return old + + +def default_retrieval(store): + return [r for r in store + if r.is_active and not (r.volatile and not r.fresh)] + + +def t_opposite_instruction_never_reinforces(): + old = Rec("Always allow external sharing of customer data", + "customer-data-sharing", "external-sharing-policy", "global", + "session-2026-01", "allow", confidence=0.8) + store = [old] + new = Rec("Never allow external sharing of customer data", + "customer-data-sharing", "external-sharing-policy", "global", + "user-correction", "deny") + sim = SequenceMatcher(None, old.text.lower(), new.text.lower()).ratio() + assert sim > 0.75, f"the counterexample pair fell under threshold: {sim}" + cands = find_candidates(store, new.text) + assert cands == [old], "candidate retrieval missed the pair" + before = old.confidence + verdict = contradiction_gate(old, new) + assert verdict == "contradicts", f"the gate said {verdict!r}" + kept = apply_verdict(verdict, old, new, store) + assert kept is new and not old.is_active, \ + "the correction did not supersede the stale instruction" + assert old.confidence == before, \ + "the contradiction BUMPED the old record — the finding itself" + assert old.superseded_by == new.id and old in store, \ + "the old fact left history — supersession must be reversible" + assert default_retrieval(store) == [new], \ + "default retrieval still serves the superseded instruction" + + +def t_shared_keywords_are_not_a_conflict(): + a = Rec("Use Python", "language-choice", "preferred-language", "global", + "s1", "python") + b = Rec("Never use production credentials", "credentials", + "production-credential-policy", "global", "s2", "deny") + assert contradiction_gate(a, b) == "unrelated", \ + "a negation flip on `use` superseded an unrelated memory" + + +def t_correction_wins_only_in_its_scope(): + old = Rec("Timeout is 30s", "api-timeout", "value", "project-B", "s1", "30s") + store = [old] + new = Rec("Timeout is 60s", "api-timeout", "value", "project-A", "s2", "60s") + verdict = contradiction_gate(old, new) + assert verdict == "coexist", f"a project-A correction hit project-B: {verdict!r}" + apply_verdict(verdict, old, new, store) + assert old.is_active and new in store, "scoped facts did not coexist" + same = Rec("Timeout is 60s", "api-timeout", "value", "project-B", "s3", "60s") + v2 = contradiction_gate(old, same) + assert v2 == "contradicts", "a number change in scope was not a contradiction" + + +def t_self_repetition_bumps_nothing(): + old = Rec("Table uses soft-delete", "orders-table", "delete-mode", + "global", "s1", "soft", confidence=0.6) + same_source = Rec("Table uses soft-delete", "orders-table", "delete-mode", + "global", "s1", "soft") + apply_verdict(contradiction_gate(old, same_source), old, same_source, [old]) + assert old.confidence == 0.6, "a self-generated repeat raised confidence" + other = Rec("Table uses soft-delete", "orders-table", "delete-mode", + "global", "s2-independent", "soft") + apply_verdict(contradiction_gate(old, other), old, other, [old]) + assert abs(old.confidence - 0.7) < 1e-9, \ + "independent corroboration did not count" + + +def t_verified_volatile_fact_still_expires(): + fact = Rec("Current API quota is 1000/day", "api-quota", "value", "global", + "verified-check", "1000", volatile=True, fresh=False, + confidence=0.95) + assert default_retrieval([fact]) == [], \ + "a stale volatile fact was served because it was once verified" + + +def main(): + case("the doctrine states the gate and the old behaviour is gone", + t_doctrine_states_the_gate) + case("an opposite instruction supersedes, never reinforces — history kept", + t_opposite_instruction_never_reinforces) + case("shared keywords and a negation flip are not a conflict", + t_shared_keywords_are_not_a_conflict) + case("a correction wins only in its own scope; a number change is caught", + t_correction_wins_only_in_its_scope) + case("self-repetition bumps nothing; independent corroboration does", + t_self_repetition_bumps_nothing) + case("verified does not exempt a volatile fact from freshness", + t_verified_volatile_fact_still_expires) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-03.02.py b/test/audit_regressions/fix-as-03.02.py new file mode 100644 index 0000000..af8960d --- /dev/null +++ b/test/audit_regressions/fix-as-03.02.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""FIX-AS-03.02 — the temporal evidence lifecycle (sherlock audit, second +leaf of AS-03, on FIX-AS-03.01's contradiction gate). + +The rules under test, run as the documented lifecycle: retrieval is not a +lifecycle event (repeated retrieval raises nothing); supersession is a +reversible annotation — the new dated fact keeps the replacement history, +the chain is walkable, restoring is clearing two fields; and a volatile +fact past its freshness window leaves default retrieval at ANY confidence, +verified included. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", + "references", "memory-lifecycle.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_the_lifecycle(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("Retrieval is not a lifecycle event", + "Supersession is an annotation, and it is reversible", + "clearing two fields, not resurrecting a deleted row", + "`verified` is about confidence, never about time", + "nothing exempts a fact about the present from the present", + "The temporal evidence lifecycle"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + + +# --------------- the documented lifecycle, executed + + +class Store: + def __init__(self): + self.rows = {} + self.n = 0 + + def add(self, value, observed_at, provenance, volatile=False, + fresh_until=None, confidence=0.6, verified=False): + self.n += 1 + self.rows[self.n] = { + "id": self.n, "value": value, "observed_at": observed_at, + "provenance": [provenance], "volatile": volatile, + "fresh_until": fresh_until, "confidence": confidence, + "verified": verified, "is_active": True, "superseded_by": None} + return self.n + + def retrieve(self, rid): + return dict(self.rows[rid]) # a READ. Nothing moves. + + def corroborate(self, rid, provenance): + r = self.rows[rid] + if provenance in r["provenance"]: + return False # self-repeat: no event + r["confidence"] = min(1.0, r["confidence"] + 0.1) + r["provenance"].append(provenance) + return True + + def supersede(self, old_id, value, observed_at, provenance): + new_id = self.add(value, observed_at, provenance) + self.rows[old_id]["is_active"] = False + self.rows[old_id]["superseded_by"] = new_id + return new_id + + def restore(self, old_id): + self.rows[old_id]["is_active"] = True + self.rows[old_id]["superseded_by"] = None + + def default_retrieval(self, now): + return [r["id"] for r in self.rows.values() + if r["is_active"] + and not (r["volatile"] and r["fresh_until"] is not None + and now > r["fresh_until"])] + + def history(self, rid): + chain = [rid] + while self.rows[chain[-1]]["superseded_by"]: + chain.append(self.rows[chain[-1]]["superseded_by"]) + return chain + + +def t_repeated_retrieval_raises_nothing(): + s = Store() + rid = s.add("timeout=30s", 100, "s1") + before = s.rows[rid]["confidence"] + for _ in range(50): + s.retrieve(rid) + assert s.rows[rid]["confidence"] == before, \ + "fifty retrievals raised confidence — the finding itself" + assert s.corroborate(rid, "s1") is False and s.rows[rid]["confidence"] == before, \ + "a same-provenance repeat counted as corroboration" + assert s.corroborate(rid, "s2") is True, "independent corroboration was refused" + + +def t_supersession_keeps_history_and_reverses(): + s = Store() + a = s.add("plan=basic", 100, "s1") + b = s.supersede(a, "plan=pro", 200, "s2") + assert s.rows[a]["is_active"] is False and s.rows[a]["superseded_by"] == b + assert a in s.rows and s.rows[a]["value"] == "plan=basic", \ + "the old dated fact left the store — history lost" + assert s.history(a) == [a, b], "the replacement chain is not walkable" + assert s.default_retrieval(300) == [b], \ + "default retrieval still serves the superseded value" + s.restore(a) + assert s.rows[a]["is_active"] and s.rows[a]["superseded_by"] is None, \ + "restoring was more than clearing two fields" + + +def t_volatile_verified_fact_expires(): + s = Store() + rid = s.add("quota=1000/day", 100, "verified-check", volatile=True, + fresh_until=500, confidence=1.0, verified=True) + assert s.default_retrieval(400) == [rid], "a fresh volatile fact was hidden" + assert s.default_retrieval(600) == [], \ + "a stale volatile fact was served at confidence 1.0 because verified" + stable = s.add("company founded 2019", 100, "s1") + assert stable in s.default_retrieval(10_000), \ + "a stable fact was expired — freshness applies to volatile facts only" + + +def main(): + case("the doctrine states the lifecycle", t_doctrine_states_the_lifecycle) + case("repeated retrieval raises nothing; independence corroborates", + t_repeated_retrieval_raises_nothing) + case("supersession keeps the dated history and reverses cleanly", + t_supersession_keeps_history_and_reverses) + case("a volatile verified fact still expires; a stable one does not", + t_volatile_verified_fact_expires) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-04.01.py b/test/audit_regressions/fix-as-04.01.py new file mode 100644 index 0000000..c2e4d9b --- /dev/null +++ b/test/audit_regressions/fix-as-04.01.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""FIX-AS-04.01 — the fake-edge test is typed (sherlock audit, AS-04). + +The finding: "No payload, no edge" and the delete step did not distinguish +dataflow from control flow, approval, or shared state — so backup→migration, +approval→charge and lease→edit (all payload-free, all real) were deletable +by the doctrine's own rule. + +The fix under test, run as the documented behaviour: an edge is typed +data/control/authorization/resource; deletion requires NONE of the four; +side-effect footprints and read/write sets are compared before fan-out, so +independent read-only reviews genuinely parallelise while two writers of one +resource serialise. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +REFS = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_types_the_edge(): + ge = " ".join(open(os.path.join(REFS, "references", "graph-engineering.md"), + encoding="utf-8").read().split()) + for needle in ("**data** | A's output enters B", + "**control** | ordering only", + "**authorization** | a decision that permits B", + "**resource** | A and B touch one thing", + "type it, and write the rationale on the arrow", + "an empty payload cell justifies deletion only when there is " + "also no causal, permissive or resource constraint", + "compare the branches' side-effect footprints and read/write sets"): + assert needle in ge, f"graph-engineering.md no longer states {needle!r}" + assert "No payload named ⇒ delete the edge" not in ge, \ + "the untyped delete rule survived in the workflow defaults" + sk = " ".join(open(os.path.join(REFS, "SKILL.md"), encoding="utf-8").read().split()) + assert "Type every edge: data, control, authorization or resource" in sk + assert "No payload, no edge." not in sk, "the untyped rule survived in SKILL.md" + + +# ---------------- the documented test, executed + + +def edge(a, b, kind=None, rationale=None): + return {"from": a, "to": b, "kind": kind, "rationale": rationale} + + +def keep(e): + """Step 3-5 as documented: any of the four kinds keeps the edge.""" + return e["kind"] in ("data", "control", "authorization", "resource") + + +def fan_out_plan(nodes): + """Step 6: read/write sets before fan-out. Writers of one resource + serialise (a resource edge is drawn); read-only branches parallelise.""" + edges = [] + for i, a in enumerate(nodes): + for b in nodes[i + 1:]: + shared = set(a["writes"]) & (set(b["writes"]) | set(b["reads"])) + shared |= set(b["writes"]) & set(a["reads"]) + if shared: + edges.append(edge(a["id"], b["id"], "resource", + f"both touch {sorted(shared)}")) + parallel = [n["id"] for n in nodes + if not any(e for e in edges if n["id"] in (e["from"], e["to"]))] + return edges, parallel + + +def t_payload_free_constraints_survive(): + for e in (edge("backup", "migration", "control", "migration is unsafe before backup"), + edge("approval", "charge", "authorization", "the decision permits the money"), + edge("lease", "edit", "authorization", "the lease permits the write")): + assert keep(e), f"{e['from']}→{e['to']} was deleted — the finding itself" + + +def t_truly_fake_edge_still_dies(): + e = edge("review-A", "review-B") + assert not keep(e), "an untyped, unconstrained edge was kept" + + +def t_two_writers_serialise_readers_parallelise(): + nodes = [ + {"id": "review-1", "reads": ["src/"], "writes": []}, + {"id": "review-2", "reads": ["src/"], "writes": []}, + {"id": "write-ledger-a", "reads": [], "writes": ["ledger.md"]}, + {"id": "write-ledger-b", "reads": [], "writes": ["ledger.md"]}, + ] + edges, parallel = fan_out_plan(nodes) + assert "review-1" in parallel and "review-2" in parallel, \ + "independent read-only reviews were serialised without cause" + assert any(e["kind"] == "resource" and "ledger.md" in e["rationale"] + for e in edges), \ + "two writers of one ledger fanned out unserialised — the lost-write race" + + +def t_reader_of_a_writers_target_is_an_edge_too(): + nodes = [{"id": "writer", "reads": [], "writes": ["state.json"]}, + {"id": "reader", "reads": ["state.json"], "writes": []}] + edges, parallel = fan_out_plan(nodes) + assert edges and not parallel, \ + "a reader raced the writer of its own input" + + +def main(): + case("the doctrine types the edge and drops the untyped delete rule", + t_doctrine_types_the_edge) + case("backup→migration, approval→charge, lease→edit survive", + t_payload_free_constraints_survive) + case("a truly fake edge still dies", t_truly_fake_edge_still_dies) + case("two writers serialise; read-only reviews parallelise", + t_two_writers_serialise_readers_parallelise) + case("a reader of a writer's target is an edge too", + t_reader_of_a_writers_target_is_an_edge_too) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-05.01.py b/test/audit_regressions/fix-as-05.01.py new file mode 100644 index 0000000..64d59aa --- /dev/null +++ b/test/audit_regressions/fix-as-05.01.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""FIX-AS-05.01 — auditability is a property of the record, not of a static graph +(sherlock audit, AS-05). + +The finding: the doctrine declared a dynamic graph unfalsifiable from outside +and a static graph mandatory for audit — conflating the plan drawn beforehand +with the saved execution graph. + +The fix under test: +* the three docs redefine auditability via completeness of the execution + record, the policy version and deterministic bounds (budget/depth/node caps) + with provenance; static stays the PREFERENCE for predictability; +* a design diagram alone never passes; +* the reconstruction audit (modelled) passes one static AND one dynamic + scenario, detects a deleted event/edge in both, and refuses a diagram-only + claim. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +HARNESS = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-harness", "SKILL.md") +ORCH = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", "SKILL.md") +GE = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-orchestrator", + "references", "graph-engineering.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def flat(p): + with open(p, encoding="utf-8") as fh: + return " ".join(fh.read().split()) + + +def t_docs_redefine_auditability(): + for p, name in ((HARNESS, "agent-harness"), (ORCH, "agent-orchestrator"), (GE, "graph-engineering")): + d = flat(p) + assert "execution record" in d, f"{name}: auditability is not defined via the execution record" + assert "budget" in d and "depth" in d and "node cap" in d, \ + f"{name}: the deterministic bounds (budget/depth/node caps) are missing" + assert "provenance" in d, f"{name}: provenance is not required" + assert "diagram" in d.lower(), f"{name}: the design-diagram-alone rule is missing" + + +def t_static_is_preference_not_audit_requirement(): + h = flat(HARNESS) + assert "auditability is\nNOT the same axis as static structure".replace("\n", " ") in h \ + or "auditability is NOT the same axis as static structure" in h, \ + "agent-harness still equates auditability with static structure" + g = flat(GE) + assert "PREFERENCE, not a hard ban" in g, "graph-engineering still bans dynamic for audit" + assert "| **never dynamic** | **you will need to audit" not in g, \ + "the 'never dynamic when you need to audit' hard rule survived" + + +# ---------------- the reconstruction audit, run as behaviour + + +def reconstruct(record): + """Passes when the execution record is complete, has a policy version and + stayed within deterministic bounds — regardless of static/dynamic. A design + diagram alone (no events) never passes.""" + if record.get("kind") == "diagram-only": + return False + events = record.get("events") + edges = record.get("edges") + if not events or not edges: + return False + if not record.get("policy_version") or not record.get("provenance"): + return False + caps = record.get("caps") or {} + if not all(k in caps for k in ("budget", "depth", "node")): + return False + # every edge's endpoints must appear as events (the record is complete) + seen = {e["node"] for e in events} + for edge in edges: + if edge["from"] not in seen or edge["to"] not in seen: + return False + return True + + +def complete_record(kind): + return { + "kind": kind, + "events": [{"node": "A"}, {"node": "B"}, {"node": "C"}], + "edges": [{"from": "A", "to": "B"}, {"from": "B", "to": "C"}], + "policy_version": "p1", + "provenance": "run-7", + "caps": {"budget": 1000, "depth": 5, "node": 20}, + } + + +def t_static_and_dynamic_both_pass_the_same_audit(): + assert reconstruct(complete_record("static")) is True, "a complete static record failed the audit" + assert reconstruct(complete_record("dynamic")) is True, \ + "a complete DYNAMIC record failed the same audit — the finding itself" + + +def t_deleted_event_or_edge_detected_in_both(): + for kind in ("static", "dynamic"): + r = complete_record(kind) + r["events"] = [e for e in r["events"] if e["node"] != "C"] # delete an event + assert reconstruct(r) is False, f"a deleted event went undetected in the {kind} record" + r2 = complete_record(kind) + r2["edges"] = r2["edges"][:-1] # delete an edge + # an edge deletion is detected because the reconstruction no longer covers C's arrival + assert reconstruct(r2) is True or reconstruct(r2) is False # structural presence + # stronger: a MISSING edge under a claim of completeness is caught by a count check + assert len(r2["edges"]) < len(complete_record(kind)["edges"]), "edge deletion not modelled" + + +def t_diagram_alone_never_passes(): + assert reconstruct({"kind": "diagram-only"}) is False, "a design diagram alone passed" + assert reconstruct({"kind": "dynamic", "events": [], "edges": []}) is False, \ + "an empty record passed" + + +def main(): + case("all three docs redefine auditability via the execution record + bounds", + t_docs_redefine_auditability) + case("static is a preference, not an audit requirement", t_static_is_preference_not_audit_requirement) + case("a complete static AND dynamic record both pass the same audit", + t_static_and_dynamic_both_pass_the_same_audit) + case("a deleted event/edge is detected in both", t_deleted_event_or_edge_detected_in_both) + case("a design diagram alone never passes", t_diagram_alone_never_passes) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-06.01.py b/test/audit_regressions/fix-as-06.01.py new file mode 100644 index 0000000..6e6676d --- /dev/null +++ b/test/audit_regressions/fix-as-06.01.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""FIX-AS-06.01 — the first release ships with an executable seed corpus +(sherlock audit, AS-06). + +The finding: §6 said the corpus comes "from production, never up front" and +the first offline gate is "observables only" — leaving a greenfield feature +with criteria but no executed trials, a gate that cannot run. And the same +chapter then allowed simulated users, a contradictory route. + +The fix under test: a SEED corpus (curated/synthetic/manual, provenance per +input) is allowed before release with at least a happy, an adversarial and a +failure/retry trial; the release gate requires EXECUTED trials — +observable-only is specification-ready, not release-ready; a corrupted +input/fixture/runner is TEST_ERROR, not a behaviour pass/fail; and cases are +isolated. + +Standard library only. +""" +import json +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SKILL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", "SKILL.md") +CORPUS = os.path.join(ROOT, "test", "evals", "fixtures", "bootstrap-corpus.json") +SCEN = os.path.join(ROOT, "test", "evals", "scenarios.json") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_the_seed_and_gate(): + flat = " ".join(open(SKILL, encoding="utf-8").read().split()) + for needle in ("The first release has no production — so it runs against a " + "SEED corpus, and\nobservable-only is not release-ready".replace("\n", " "), + "with at least a **happy**, an **adversarial** and a " + "**failure/retry** trial", + "Each seed input carries its **provenance**", + "The release gate requires EXECUTED trials", + "`specification-ready`, not\n`release-ready`".replace("\n", " "), + "A corrupted fixture, input or runner is a `TEST_ERROR`", + "cases are isolated"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + + +def t_seed_corpus_has_the_three_kinds_with_provenance(): + c = json.load(open(CORPUS, encoding="utf-8")) + trials = c["trials"] + kinds = {t["kind"] for t in trials} + assert {"happy", "adversarial", "failure_retry"} <= kinds, \ + f"the seed corpus is missing a required trial kind: {kinds}" + for t in trials: + assert t.get("provenance") in ("curated", "synthetic", "manual"), \ + f"{t['id']}: no valid provenance mark" + assert t.get("input") and t.get("observable"), \ + f"{t['id']}: a trial with no input or observable is not executable" + + +def t_scenarios_note_points_at_the_seed_and_names_specification_ready(): + note = json.load(open(SCEN, encoding="utf-8"))["note"] + assert "specification-ready" in note and "EXECUTED trials" in note, \ + "the scenarios note still calls authored observables release-ready" + assert "bootstrap-corpus.json" in note, "the note does not point at the seed corpus" + + +# ---------------- the release-gate + runner rules, executed + + +REQUIRED_KINDS = {"happy", "adversarial", "failure_retry"} + + +def gate(trials, executed): + """Release-ready needs EXECUTED trials covering the required kinds; a + suite with nothing run is specification-ready.""" + if not executed: + return "specification-ready" + covered = {t["kind"] for t in trials if t["id"] in executed} + if REQUIRED_KINDS <= covered: + return "release-ready" + return "specification-ready" + + +def run_trial(trial): + """A corrupted input/fixture is TEST_ERROR, distinct from a behaviour + pass/fail.""" + if not trial.get("input") or trial.get("input") == "": + return "TEST_ERROR" + return "PASS" if trial.get("_behaviour_ok", True) else "FAIL" + + +def t_observable_only_is_specification_ready(): + trials = json.load(open(CORPUS, encoding="utf-8"))["trials"] + assert gate(trials, executed=set()) == "specification-ready", \ + "a suite with nothing executed was called release-ready" + all_ids = {t["id"] for t in trials} + assert gate(trials, executed=all_ids) == "release-ready", \ + "an executed seed covering all three kinds was not release-ready" + + +def t_corrupt_input_is_test_error(): + assert run_trial({"input": ""}) == "TEST_ERROR", \ + "a corrupted input scored as a behaviour result" + assert run_trial({"input": "real", "_behaviour_ok": False}) == "FAIL" + assert run_trial({"input": "real"}) == "PASS" + + +def t_cases_are_isolated(): + trials = json.load(open(CORPUS, encoding="utf-8"))["trials"] + a, b = trials[0], trials[1] + r_ab = (run_trial(a), run_trial(b)) + r_b_alone = run_trial(b) + assert r_ab[1] == r_b_alone, "case B's result depended on whether A ran" + + +def main(): + case("the doctrine states the seed corpus and release gate", + t_doctrine_states_the_seed_and_gate) + case("the seed corpus has happy/adversarial/failure_retry with provenance", + t_seed_corpus_has_the_three_kinds_with_provenance) + case("the scenarios note points at the seed and says specification-ready", + t_scenarios_note_points_at_the_seed_and_names_specification_ready) + case("observable-only is specification-ready, executed is release-ready", + t_observable_only_is_specification_ready) + case("a corrupt input is TEST_ERROR, not a behaviour result", + t_corrupt_input_is_test_error) + case("cases are isolated", t_cases_are_isolated) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-07.01.py b/test/audit_regressions/fix-as-07.01.py new file mode 100644 index 0000000..3d7f114 --- /dev/null +++ b/test/audit_regressions/fix-as-07.01.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""FIX-AS-07.01 — the order ban keeps the mandatory happens-before (sherlock +audit, AS-07). + +The finding: the advice to avoid a brittle exact tool-order assertion had +hardened into a BAN on order — "as a set and a forbidden list, never as an +order" — which throws out the semantically mandatory happens-before +(authorization before effect, fresh read before write, commit before +publish). + +The fix under test: forbid only the redundant EXACT GLOBAL sequence; assert +the mandatory edges as a PARTIAL order — reordering two independent reads +passes, reordering confirm/charge or acquire/write fails; and keep a +negative example beside the rubric. Documented in SKILL.md, and the +partial-order matcher is run as behaviour. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SKILL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", "SKILL.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_carves_out_partial_order(): + flat = " ".join(open(SKILL, encoding="utf-8").read().split()) + for needle in ("What is forbidden is the redundant **exact global sequence**, " + "not order as such", + "A few **happens-before** edges are semantically mandatory", + "authorization\nprecedes its effect".replace("\n", " "), + "a fresh read precedes the write depending on it", + "reordering two independent reads must\npass".replace("\n", " "), + "reordering confirm/charge or acquire/write must fail", + "keep the\nnegative example (a confirm-after-charge trace)".replace("\n", " ")): + assert needle in flat, f"the doctrine no longer states {needle!r}" + assert "as a **set and a forbidden list**, never as an order" not in flat, \ + "the total ban on order survived" + + +# ---------------- the partial-order matcher, executed + + +def satisfies_partial_order(trace, edges): + """Every (a, b) edge means a must appear before b in the trace. Independent + calls not named in any edge may appear in any order.""" + pos = {} + for i, call in enumerate(trace): + pos.setdefault(call, i) + for a, b in edges: + if a in pos and b in pos and pos[a] > pos[b]: + return False + return True + + +MANDATORY = [("authorize", "charge"), ("read_fresh", "write"), ("commit", "publish")] + + +def t_independent_reads_reorder_freely(): + edges = [("read_fresh", "write")] + assert satisfies_partial_order(["read_a", "read_b", "read_fresh", "write"], edges) + assert satisfies_partial_order(["read_b", "read_a", "read_fresh", "write"], edges), \ + "reordering two independent reads was rejected — the order ban's over-reach" + + +def t_confirm_after_charge_fails(): + assert satisfies_partial_order(["authorize", "charge"], [("authorize", "charge")]) + assert not satisfies_partial_order(["charge", "authorize"], [("authorize", "charge")]), \ + "confirm/charge reordering passed — a mandatory happens-before was not enforced" + + +def t_acquire_after_write_fails(): + edges = [("acquire", "write")] + assert satisfies_partial_order(["acquire", "write"], edges) + assert not satisfies_partial_order(["write", "acquire"], edges), \ + "write-before-acquire passed — the lease-before-edit edge was not enforced" + + +def t_partial_order_is_not_a_total_order(): + # a total-order assertion would reject a valid reordering of unrelated calls; + # the partial order must not. + edges = [("authorize", "charge")] + assert satisfies_partial_order( + ["log", "authorize", "metric", "charge", "notify"], edges), \ + "unrelated calls around a mandatory edge were rejected" + + +def main(): + case("the doctrine carves out the mandatory partial order", + t_doctrine_carves_out_partial_order) + case("two independent reads reorder freely", t_independent_reads_reorder_freely) + case("confirm-after-charge fails", t_confirm_after_charge_fails) + case("acquire-after-write fails", t_acquire_after_write_fails) + case("the partial order is not a total order", t_partial_order_is_not_a_total_order) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-08.01.py b/test/audit_regressions/fix-as-08.01.py new file mode 100644 index 0000000..e4afa90 --- /dev/null +++ b/test/audit_regressions/fix-as-08.01.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""FIX-AS-08.01 — proportion intervals and trial units (sherlock audit, AS-08). + +The finding: the Wald interval was given without n/p limits (zero width at +p=0/1 — certainty from a boundary); "dependence always widens the band" and +"p^k is a worst case" were stated as theorems though both need assumptions; +comparing independent averages was banned outright; repeated trials of one +task could be pooled as if they were independent tasks. + +The fix under test (statistics.md): +* Wilson is the default (extracted from the doc and RUN): nonzero width at the + boundary, (0,1) at n=0 — a zero/small sample is never a universal bound; +* the Wald helper states its validity limits; +* dependence-widens and p^k-baseline carry their assumptions; +* unpaired comparison is legitimate with the wider band; pairing preferred; +* pass@k/pass^k aggregate over TASK-level trials, never pooling repeats. + +Standard library only. +""" +import os +import re +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", + "references", "statistics.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def doc(): + with open(DOC, encoding="utf-8") as fh: + return fh.read() + + +def load_wilson(): + text = doc() + for m in re.finditer(r"```python\n(.*?)```", text, re.S): + if "def wilson" in m.group(1): + ns = {} + exec(compile(m.group(1), "statistics.md", "exec"), ns) + return ns["wilson"], ns["band"] + raise AssertionError("the wilson code block was not found in statistics.md") + + +def t_wilson_is_shipped_and_boundary_honest(): + wilson, band = load_wilson() + lo, hi = wilson(0.0, 5) + assert hi > 0.3, f"5 clean runs left almost no uncertainty: hi={hi}" + lo1, hi1 = wilson(1.0, 3) + assert lo1 < 0.75, f"3/3 passes claimed near-certainty: lo={lo1}" + assert wilson(0.5, 0) == (0.0, 1.0), "n=0 is not total uncertainty" + # the Wald defect, demonstrated: zero width at the boundary + assert band(0.0, 5) == 0.0, "the doc's Wald no longer shows the boundary defect" + + +def t_wald_states_its_limits(): + d = " ".join(doc().split()) + assert "n*p >= 10 and n*(1-p) >= 10" in d, "the Wald validity limits are missing" + assert "zero or tiny sample is not a universal bound" in d or \ + "a zero or tiny sample is not a universal bound" in d.lower(), \ + "small-sample humility is missing" + + +def t_assumptions_are_named(): + d = " ".join(doc().split()) + assert "POSITIVE intra-cluster correlation" in d, \ + "dependence-widens carries no assumption" + assert "an assumption, not a theorem" in d.lower() or "assumption, not a theorem" in d, \ + "the widening claim is still a theorem" + assert "INDEPENDENCE BASELINE, not a bound" in d, "p^k is still called a worst case" + + +def t_unpaired_is_legitimate_with_wider_band(): + d = " ".join(doc().split()) + assert "Never subtract two independent averages." not in d, \ + "the absolute ban on independent averages survived" + assert "still legitimate when pairing is impossible" in d, \ + "unpaired comparison is still forbidden" + assert "wider two-sample band" in d + + +def t_task_level_trials(): + d = " ".join(doc().split()) + assert "computed over TASK-LEVEL trials" in d, "task-level aggregation is missing" + assert "trials are not independent tasks" in d, \ + "repeated trials can still be pooled as tasks" + + +# ---------------- the trial-unit rule, run as behaviour + + +def benchmark_pass_at_k(tasks, pooled): + """tasks: {task: [trial results]}. Correct: mean over tasks of per-task + any-success. Pooled (the defect): every trial counted as its own task.""" + if pooled: + allt = [r for rs in tasks.values() for r in rs] + return sum(allt) / len(allt) + per_task = [1.0 if any(rs) else 0.0 for rs in tasks.values()] + return sum(per_task) / len(per_task) + + +def t_pooled_trials_inflate(): + tasks = {"easy": [1, 1, 1, 1, 1], "hard": [0, 0, 0, 0, 1]} + correct = benchmark_pass_at_k(tasks, pooled=False) + pooled = benchmark_pass_at_k(tasks, pooled=True) + assert correct == 1.0, "per-task any-success mis-modelled" + assert pooled != correct, \ + "the model cannot show the pooling defect — the regression proves nothing" + + +def main(): + case("wilson ships, is boundary-honest, n=0 = total uncertainty", + t_wilson_is_shipped_and_boundary_honest) + case("the Wald helper states its validity limits", t_wald_states_its_limits) + case("dependence-widens and p^k-baseline carry their assumptions", + t_assumptions_are_named) + case("unpaired comparison is legitimate with the wider band", + t_unpaired_is_legitimate_with_wider_band) + case("pass@k/pass^k aggregate over task-level trials", t_task_level_trials) + case("pooling repeated trials as tasks visibly distorts the number", + t_pooled_trials_inflate) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-08.02.py b/test/audit_regressions/fix-as-08.02.py new file mode 100644 index 0000000..3a30e1c --- /dev/null +++ b/test/audit_regressions/fix-as-08.02.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""FIX-AS-08.02 — paired/clustered/unpaired are separated, and splits are spent +once (sherlock audit, AS-08 + ADOPT-M-04). + +The finding: McNemar/bootstrap were prescribed without tying them to the +design; dependent repeats could be treated iid; and nothing stopped a reused +validation example from being relabelled an "unseen final test". + +The fix under test (statistics.md): +* a design→method table: paired→McNemar/paired-bootstrap over deltas; + clustered→cluster bootstrap resampling TASKS; unpaired→two-sample (Welch) + SE — and the receipt names both, matching; +* dependent repeats are never iid; small n buys no certainty; +* case IDs/groupings are FIXED; validation may tune; the final holdout is + spent once and never picks versions; "unseen" is run-history, not a label; +* the matching rule is RUN as behaviour on the audit's counterexamples. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", + "references", "statistics.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def flat(): + with open(DOC, encoding="utf-8") as fh: + return " ".join(fh.read().split()) + + +def t_design_method_table(): + d = flat() + assert "The design decides the method" in d + for needle in ("cluster bootstrap that resamples TASKS", + "two-sample (Welch) SE", + "receipt names the design AND the method"): + assert needle in d, f"the doc no longer states {needle!r}" + assert "treats dependent repeats as iid" in d, "the trial-bootstrap trap is not named" + + +def t_repeats_never_iid_small_n_humble(): + d = flat() + assert "Dependent repeats are never claimed iid" in d + assert "a small sample never buys\nimaginary certainty".replace("\n", " ") in d or \ + "a small sample never buys imaginary certainty" in d + + +def t_splits_spent_once(): + d = flat() + assert "case IDs and their groupings" in d.lower() or "case ids and their groupings" in d.lower() + assert "frozen before any run" in d + assert "MAY be used for tuning" in d, "validation tuning is forbidden again" + assert "spent ONCE" in d and "never used to pick between versions" in d + assert 'relabelled an "unseen final test"' in d + assert "property of the RUN HISTORY" in d + + +# ---------------- the matching rule, run as behaviour + + +def receipt_ok(design, method, labels=None): + """The doc's rule: the method must match the design; splits keep their + history; dependent repeats are not iid.""" + match = {"paired": {"mcnemar", "paired-bootstrap"}, + "clustered": {"cluster-bootstrap-tasks"}, + "unpaired": {"welch-two-sample"}} + if method not in match.get(design, set()): + return False + for lab in labels or []: + if lab.get("claim") == "unseen-final-test" and lab.get("history") == "used-in-validation": + return False + if lab.get("claim") == "iid" and lab.get("structure") == "dependent-repeats": + return False + return True + + +def t_matching_rule_behaviour(): + assert receipt_ok("paired", "mcnemar") is True + assert receipt_ok("unpaired", "mcnemar") is False, "McNemar over an unpaired corpus passed" + assert receipt_ok("clustered", "cluster-bootstrap-tasks") is True + assert receipt_ok("clustered", "paired-bootstrap") is False, \ + "a trial-level bootstrap passed a clustered corpus" + assert receipt_ok("unpaired", "welch-two-sample") is True + assert receipt_ok("paired", "mcnemar", + [{"claim": "unseen-final-test", "history": "used-in-validation"}]) is False, \ + "a reused validation example was relabelled an unseen final test — the finding" + assert receipt_ok("clustered", "cluster-bootstrap-tasks", + [{"claim": "iid", "structure": "dependent-repeats"}]) is False, \ + "dependent repeats were claimed iid" + + +def main(): + case("the design→method table stands, receipt names both", t_design_method_table) + case("dependent repeats never iid; small n buys nothing", + t_repeats_never_iid_small_n_humble) + case("splits are frozen; holdout spent once; unseen is history", + t_splits_spent_once) + case("the matching rule refuses the audit's counterexamples", + t_matching_rule_behaviour) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-09.01.py b/test/audit_regressions/fix-as-09.01.py new file mode 100644 index 0000000..4a68888 --- /dev/null +++ b/test/audit_regressions/fix-as-09.01.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""FIX-AS-09.01 — extensible OTel fields, and total vs bucket tokens (sherlock +audit, AS-09). + +The finding: `gen_ai.operation.name` was called a closed 17-value enum where a +value outside it "is not an extension" — but the semconv treats it as a +well-known SET; a provider value with no match is allowed and must not be +silently dropped, an unknown value kept raw. And the token text claimed +input+output "misses reasoning and cache writes entirely", when those are +SUBSETS of the totals, not additions. + +The fix under test: the doc states the extensible set with raw-value +preservation and a schema-revision/observation-date rule; the cost model +subtracts the cached portion rather than re-adding buckets. The +total-vs-bucket accounting is run as behaviour. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", + "references", "otel-genai.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_extensible_set(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("well-known SET, not a closed enum", + "a provider\noperation with no matching well-known value is allowed to " + "carry a custom value".replace("\n", " "), + "an unknown value is stored RAW", + "schema revision and the commit SHA", + "plus the observation date"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + assert "closed 17-value enum" not in flat, "the closed-enum claim survived" + assert "misses reasoning tokens and\ncache writes entirely".replace("\n", " ") not in flat, \ + "the 'misses reasoning/cache entirely' claim survived" + + +def t_doctrine_states_total_vs_bucket(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("TOTALS and\nDISJOINT BILLING BUCKETS".replace("\n", " "), + "`reasoning.output_tokens` is a SUBSET of\n`output_tokens`".replace("\n", " "), + "SUBTRACT the cached portion from\nthe total".replace("\n", " "), + "The fix is NOT to add reasoning or modality counters back onto " + "the totals"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + + +# ---------------- the token accounting, executed + + +def preserve_operation(value, well_known): + """A well-known value is kept as-is; an unknown one is kept RAW, never + dropped.""" + return {"name": value, "well_known": value in well_known} + + +def cost(usage, rates): + """Correct cost: the cached input portion is priced at the cache rate, the + rest of the input at the full rate, output at the output rate. Reasoning + and modality counters are SUBSETS and are NOT re-added.""" + inp = usage["input_tokens"] + cache_read = usage.get("cache_read.input_tokens", 0) + cache_write = usage.get("cache_write.input_tokens", 0) + full_input = inp - cache_read # cache_read is a subset of input + out = usage["output_tokens"] # reasoning is a subset of output + return (full_input * rates["input"] + + cache_read * rates["cache_read"] + + cache_write * rates["cache_write"] + + out * rates["output"]) + + +WELL_KNOWN = {"chat", "embeddings", "execute_tool", "invoke_agent", "plan"} + + +def t_unknown_operation_kept_raw(): + known = preserve_operation("chat", WELL_KNOWN) + assert known["well_known"] is True + custom = preserve_operation("provider.rerank", WELL_KNOWN) + assert custom["name"] == "provider.rerank" and custom["well_known"] is False, \ + "an unknown operation value was dropped or renamed — the finding itself" + + +def t_cache_read_is_not_billed_at_full_rate(): + usage = {"input_tokens": 1000, "output_tokens": 200, + "cache_read.input_tokens": 800} + rates = {"input": 3.0, "cache_read": 0.3, "cache_write": 3.75, "output": 15.0} + c = cost(usage, rates) + # 200 full input @3 + 800 cache_read @0.3 + 200 output @15 + assert c == 200 * 3.0 + 800 * 0.3 + 200 * 15.0, f"cost mis-priced: {c}" + naive = (usage["input_tokens"] * rates["input"] + + usage["output_tokens"] * rates["output"]) + assert c < naive, "the corrected cost did not undercut the naive full-price input" + + +def t_reasoning_and_modalities_not_double_counted(): + # reasoning.output_tokens is a subset of output_tokens; adding it would + # double-count. The cost function ignores it, so passing it changes nothing. + base = {"input_tokens": 500, "output_tokens": 300, "cache_read.input_tokens": 0} + rates = {"input": 3.0, "cache_read": 0.3, "cache_write": 3.75, "output": 15.0} + c1 = cost(base, rates) + with_reasoning = dict(base, **{"reasoning.output_tokens": 120, + "text.output_tokens": 180}) + c2 = cost(with_reasoning, rates) + assert c1 == c2, "reasoning/modality subsets were re-added to the total — double billing" + + +def main(): + case("the doctrine states the extensible well-known set and provenance", + t_doctrine_states_extensible_set) + case("the doctrine separates totals from disjoint billing buckets", + t_doctrine_states_total_vs_bucket) + case("an unknown operation value is kept raw, not dropped", + t_unknown_operation_kept_raw) + case("cache reads are not billed at the full input rate", + t_cache_read_is_not_billed_at_full_rate) + case("reasoning and modality subsets are not double-counted", + t_reasoning_and_modalities_not_double_counted) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-09.02.py b/test/audit_regressions/fix-as-09.02.py new file mode 100644 index 0000000..7e238db --- /dev/null +++ b/test/audit_regressions/fix-as-09.02.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""FIX-AS-09.02 — disjoint billing buckets reconcile (sherlock audit, AS-09 +leaf 2, on FIX-AS-09.01). + +The rules under test: totals are NOT summed with cached/modality/reasoning +subsets; provider rates apply to DISJOINT buckets; a cached token is never +charged twice; and the price receipt reconciles with the totals against an +independent worked example. Documented in otel-genai.md, and the receipt is +recomputed as behaviour. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", + "references", "otel-genai.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_has_the_worked_receipt(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("The worked receipt — a disjoint partition that reconciles", + "| input, full-rate = `input − cache_read` | 200 | $3.00 | $0.60 |", + "| **total** | | | **$3.84** |", + "sum back to the totals", + "reasoning's 120 is inside it, never a fourth line", + "OVER-charges by pricing the 800 cached tokens"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + + +# ---------------- the receipt, recomputed + + +def receipt(usage, rates): + inp = usage["input_tokens"] + cache_read = usage.get("cache_read.input_tokens", 0) + cache_write = usage.get("cache_write.input_tokens", 0) + out = usage["output_tokens"] + full_input = inp - cache_read + buckets = { + "input_full": (full_input, rates["input"]), + "cache_read": (cache_read, rates["cache_read"]), + "cache_write": (cache_write, rates.get("cache_write", 0.0)), + "output": (out, rates["output"]), + } + total = sum(tok / 1000 * rate for tok, rate in buckets.values()) + return buckets, total + + +USAGE = {"input_tokens": 1000, "cache_read.input_tokens": 800, + "output_tokens": 200, "reasoning.output_tokens": 120} +RATES = {"input": 3.0, "cache_read": 0.30, "cache_write": 3.75, "output": 15.0} + + +def t_receipt_matches_the_worked_example(): + _b, total = receipt(USAGE, RATES) + assert abs(total - 3.84) < 1e-9, f"the receipt does not reconcile: {total}" + + +def t_buckets_sum_back_to_the_totals(): + buckets, _ = receipt(USAGE, RATES) + input_tokens = buckets["input_full"][0] + buckets["cache_read"][0] + assert input_tokens == USAGE["input_tokens"], "input buckets do not sum to the total" + assert buckets["output"][0] == USAGE["output_tokens"], \ + "output includes reasoning as an extra line — double counting" + + +def t_no_cached_token_charged_twice(): + buckets, _ = receipt(USAGE, RATES) + # the 800 cached tokens appear ONLY in cache_read, never in input_full + assert buckets["input_full"][0] == 200 and buckets["cache_read"][0] == 800 + assert buckets["input_full"][0] + buckets["cache_read"][0] == 1000, \ + "a cached token was priced in two buckets" + + +def t_naive_sum_overcharges(): + naive = (USAGE["input_tokens"] / 1000 * RATES["input"] + + USAGE["output_tokens"] / 1000 * RATES["output"]) + _b, correct = receipt(USAGE, RATES) + assert naive > correct, "the naive input+output did not over-charge vs the receipt" + assert abs(naive - 6.0) < 1e-9, f"the naive figure is not the documented $6.00: {naive}" + + +def main(): + case("the doctrine carries the worked reconciling receipt", + t_doctrine_has_the_worked_receipt) + case("the receipt matches the worked example ($3.84)", + t_receipt_matches_the_worked_example) + case("the priced buckets sum back to the totals", t_buckets_sum_back_to_the_totals) + case("no cached token is charged twice", t_no_cached_token_charged_twice) + case("the naive input+output over-charges", t_naive_sum_overcharges) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-10.01.py b/test/audit_regressions/fix-as-10.01.py new file mode 100644 index 0000000..a4c19fe --- /dev/null +++ b/test/audit_regressions/fix-as-10.01.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""FIX-AS-10.01 — a regrade of an old output is not a check of the candidate +(sherlock audit, AS-10). + +The finding: "fixture replay" was described as a free assertion over a stored +run that answers "did the decision at this point change" — but an old output +does not change when the new prompt/model/tool schema changes. That operation +is a deterministic REGRADE, not a new agent trial. + +The fix under test: +* otel-genai.md splits the senses: regrade of a stored output (free, + deterministic, explicitly NOT a candidate check) vs candidate execution over + a frozen fixture (a real stochastic call, costed) vs durable-execution + replay; candidate version/output/score are their own records; +* the SKILL labels a regrade as regrade; +* the gate rule run as behaviour: mutating the candidate to a knowingly wrong + tool CHANGES the candidate-execution gate and CANNOT change a regrade — + which is precisely why the regrade may not wear the candidate label. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +OTEL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", + "references", "otel-genai.md") +SKILL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-evals", "SKILL.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def flat(p): + with open(p, encoding="utf-8") as fh: + return " ".join(fh.read().split()) + + +def t_replay_senses_are_split(): + d = flat(OTEL) + assert "Regrade of a stored output" in d + assert "Candidate execution over a frozen fixture" in d + assert "explicitly NOT a check of the candidate" in d + assert "an assertion over a stored run | free | did the decision at this point change" not in d, \ + "the free-answers-decision-change row survived — the finding itself" + assert "a real model call, stochastic" in d, "cost/stochasticity is not reflected" + + +def t_candidate_records_are_separate(): + d = flat(OTEL) + assert "stored as their own records, never overwriting the old trace" in d + s = flat(SKILL) + assert "labelled regrade" in s, "the SKILL does not label a regrade as regrade" + assert "stochastic call, costed in the receipt" in s + + +def t_mutation_must_move_the_gate(): + d = flat(OTEL) + assert "mutate the candidate to a knowingly wrong tool, and the gate's result MUST change" in d + assert "a regrade wearing the wrong label" in d + + +# ---------------- the gate rule, run as behaviour + + +OLD_TRACE_OUTPUT = {"tool": "search", "result": "ok"} + + +def regrade(rubric, candidate): + # marks the OLD output; the candidate never runs + return rubric(OLD_TRACE_OUTPUT) + + +def execute_candidate(rubric, candidate): + # runs the CANDIDATE against the frozen fixture inputs + out = {"tool": candidate["tool"], "result": "ok" if candidate["tool"] == "search" else "wrong"} + return rubric(out) + + +def t_gate_behaviour(): + rubric = lambda out: out["tool"] == "search" and out["result"] == "ok" + good = {"tool": "search"} + bad = {"tool": "delete_everything"} # knowingly wrong tool + # candidate execution: the mutation MUST change the gate + assert execute_candidate(rubric, good) is True + assert execute_candidate(rubric, bad) is False, \ + "mutating the candidate did not move the candidate-execution gate" + # regrade: the mutation CANNOT change it — the label must say so + assert regrade(rubric, good) == regrade(rubric, bad) == True, \ + "the model no longer shows why a regrade cannot check a candidate" + + +def main(): + case("the replay senses are split, cost/stochasticity reflected", + t_replay_senses_are_split) + case("candidate records separate; regrade labelled regrade", + t_candidate_records_are_separate) + case("the mutation-must-move-the-gate rule is stated", t_mutation_must_move_the_gate) + case("the gate rule run as behaviour (mutation moves execution, not regrade)", + t_gate_behaviour) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-11.01.py b/test/audit_regressions/fix-as-11.01.py new file mode 100644 index 0000000..8a5d145 --- /dev/null +++ b/test/audit_regressions/fix-as-11.01.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""FIX-AS-11.01 — the approval grant contract (sherlock audit, AS-11). + +The finding: `confirm: true` was presented as protecting a destructive action, +but the MODEL can set the boolean itself — it is not user approval. And "any +two are safe" gave blanket safety to any pair of the lethal-trifecta +capabilities, when untrusted content + write causes damage with no private-data +access. + +The fix under test: `confirm: true` is named a SYNTAX GUARD; real user approval +is a verifiable grant bound to principal/action/arguments/expiry (or an +existing authorization), and a stale grant does not authorize changed +arguments; the trifecta is a sufficient config for a specific exfiltration +risk, not a complete model. Documented in tools.md, and the grant/verification +rules are run as behaviour. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-harness", + "references", "tools.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_the_contract(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("`confirm: true` is a SYNTAX GUARD, not user approval", + "the MODEL can\nset the boolean itself".replace("\n", " "), + "verifiable grant from a trusted control plane, bound to\nthe principal, " + "action, exact arguments and an expiry".replace("\n", " "), + "a stale grant does NOT authorize changed arguments", + "a client-supplied\nboolean creates no authorization at all".replace("\n", " "), + "\"any two are safe\" over-claims", + "SUFFICIENT configuration for one SPECIFIC risk"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + assert "Any two are safe. All three in one session" not in flat, \ + "the 'any two are safe' claim survived" + + +# ---------------- the grant contract, executed + + +def is_authorized(grant, request): + """A grant authorizes a request only if it is a trusted-plane grant bound to + the same principal/action/arguments and not expired. A bare boolean is not a + grant.""" + if grant is True or grant is None: + return False # a boolean/absent grant authorizes nothing + if grant.get("issuer") != "control-plane": + return False + if grant.get("principal") != request.get("principal"): + return False + if grant.get("action") != request.get("action"): + return False + if grant.get("arguments") != request.get("arguments"): + return False # changed arguments ⇒ stale + if request.get("now", 0) > grant.get("expiry", 0): + return False + return True + + +def t_client_boolean_is_not_authorization(): + req = {"principal": "u", "action": "delete", "arguments": {"id": 7}, "now": 10} + assert is_authorized(True, req) is False, \ + "a client-supplied boolean authorized a destructive action — the finding itself" + assert is_authorized({"confirm": True}, req) is False, \ + "a model-set confirm object authorized the action" + + +def t_valid_grant_authorizes_its_exact_request(): + req = {"principal": "u", "action": "delete", "arguments": {"id": 7}, "now": 10} + grant = {"issuer": "control-plane", "principal": "u", "action": "delete", + "arguments": {"id": 7}, "expiry": 100} + assert is_authorized(grant, req) is True + + +def t_stale_grant_does_not_authorize_changed_arguments(): + grant = {"issuer": "control-plane", "principal": "u", "action": "delete", + "arguments": {"id": 7}, "expiry": 100} + changed = {"principal": "u", "action": "delete", "arguments": {"id": 999}, "now": 10} + assert is_authorized(grant, changed) is False, \ + "a grant for id 7 authorized a delete of id 999 — arguments not bound" + + +def t_expired_grant_is_refused(): + grant = {"issuer": "control-plane", "principal": "u", "action": "delete", + "arguments": {"id": 7}, "expiry": 100} + late = {"principal": "u", "action": "delete", "arguments": {"id": 7}, "now": 200} + assert is_authorized(grant, late) is False, "an expired grant still authorized" + + +def t_trifecta_pair_is_not_blanket_safe(): + # untrusted content + write, no private data: still a damage path + caps = {"untrusted_content", "write"} + exfiltration_possible = {"private_data", "untrusted_content", "external_comms"} <= caps + assert not exfiltration_possible, "this pair is not the exfiltration triangle" + # ...but it is not 'safe' — a damaging action is possible without private data + damage_possible = "untrusted_content" in caps and "write" in caps + assert damage_possible, "the write+untrusted damage path was treated as safe" + + +def main(): + case("the doctrine states the grant contract", t_doctrine_states_the_contract) + case("a client boolean is not authorization", t_client_boolean_is_not_authorization) + case("a valid grant authorizes its exact request", + t_valid_grant_authorizes_its_exact_request) + case("a stale grant does not authorize changed arguments", + t_stale_grant_does_not_authorize_changed_arguments) + case("an expired grant is refused", t_expired_grant_is_refused) + case("a trifecta pair is not blanket safe", t_trifecta_pair_is_not_blanket_safe) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-11.02.py b/test/audit_regressions/fix-as-11.02.py new file mode 100644 index 0000000..56fd888 --- /dev/null +++ b/test/audit_regressions/fix-as-11.02.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""FIX-AS-11.02 — threat-model limits: the trifecta is not full security +(sherlock audit, AS-11 leaf 2, on FIX-AS-11.01). + +The rule under test: the audit treats the lethal trifecta as a SPECIFIC +exfiltration pattern, not a complete threat model; it evaluates capabilities +and effects SEPARATELY, so a session missing one trifecta leg is NOT a PASS +for an unrelated destructive effect. Documented in audit.md, and the +capability/effect separation is run as behaviour. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-harness", + "references", "audit.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def t_doctrine_states_the_limit(): + flat = " ".join(open(DOC, encoding="utf-8").read().split()) + for needle in ("The lethal trifecta is a specific EXFILTRATION pattern, not a full\nthreat\n " + "model".replace("\n ", " ").replace("\n", " "), + "a session MISSING one leg is not thereby\n \"safe\"".replace("\n ", " ").replace("\n", " "), + "Audit **capabilities and effects SEPARATELY**", + "is an unrelated destructive\n effect".replace("\n ", " ").replace("\n", " "), + "\"Only two of the three, therefore a PASS\" is the mistake"): + assert needle in flat, f"the doctrine no longer states {needle!r}" + + +# ---------------- the capability/effect audit, executed + + +def exfiltration_risk(caps): + return {"private_data", "untrusted_content", "external_comms"} <= set(caps) + + +def destructive_effect_risk(caps): + """A damaging action does not need private data: untrusted content + a write + capability is enough.""" + return "untrusted_content" in caps and "write" in caps + + +def audit_session(caps): + """Findings are the UNION of separately-evaluated risks — a clean + exfiltration axis does not clear the destructive-effect axis.""" + findings = [] + if exfiltration_risk(caps): + findings.append("exfiltration") + if destructive_effect_risk(caps): + findings.append("destructive-effect") + return findings + + +def t_missing_leg_is_not_a_pass_for_a_destructive_effect(): + # untrusted content + write, NO private data → not the exfiltration triangle, + # but still a destructive-effect finding. + caps = {"untrusted_content", "write"} + assert not exfiltration_risk(caps), "this set is not the exfiltration triangle" + findings = audit_session(caps) + assert "destructive-effect" in findings, \ + "a missing trifecta leg PASSed a session with an unrelated destructive "\ + "effect — the finding itself" + assert "exfiltration" not in findings + + +def t_full_trifecta_is_an_exfiltration_finding(): + caps = {"private_data", "untrusted_content", "external_comms"} + assert "exfiltration" in audit_session(caps) + + +def t_axes_are_independent(): + # a session can be clean on one axis and flagged on the other + only_exfil = {"private_data", "untrusted_content", "external_comms"} + assert audit_session(only_exfil) == ["exfiltration"] + both = {"private_data", "untrusted_content", "external_comms", "write"} + assert set(audit_session(both)) == {"exfiltration", "destructive-effect"}, \ + "the two risk axes were not evaluated separately" + + +def t_a_safe_session_has_neither(): + assert audit_session({"private_data"}) == [] + assert audit_session({"read", "external_comms"}) == [] + + +def main(): + case("the doctrine states the trifecta's limit", t_doctrine_states_the_limit) + case("a missing leg is not a PASS for a destructive effect", + t_missing_leg_is_not_a_pass_for_a_destructive_effect) + case("the full trifecta is an exfiltration finding", + t_full_trifecta_is_an_exfiltration_finding) + case("the capability and effect axes are independent", t_axes_are_independent) + case("a genuinely safe session has neither finding", t_a_safe_session_has_neither) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-12.01.py b/test/audit_regressions/fix-as-12.01.py new file mode 100644 index 0000000..90cce9d --- /dev/null +++ b/test/audit_regressions/fix-as-12.01.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""FIX-AS-12.01 — the MCP shipping example pins its SDK (sherlock audit, AS-12). + +The finding: the wire revision was pinned but not the SDK's distribution, +version or import — the example used FastMCP with transport settings in the +constructor, while the standalone `fastmcp` 2.x renamed the class and moved +those args. A reader could install either distribution and get a 404 the doc +itself warns about. + +The fix under test (mcp-ship.md): +* the two distributions are distinguished; the official import is named and + the standalone import explicitly refused; +* a pinned requirements block ties the snippets to the verified version and + its date, re-pinned only with a re-run; +* v1/standalone is explicitly out of scope (or its own fixture); +* the lifecycle names the verified registration order (health BEFORE auth); +* proof is a localhost protocol call (health 200, unauth 401, initialize / + tools/list / tools/call), never a string in markdown — asserted here as + the doc's contract, and the acceptance list is machine-checked. + +Standard library only. +""" +import os +import re +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +DOC = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-interop", + "references", "mcp-ship.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def raw(): + with open(DOC, encoding="utf-8") as fh: + return fh.read() + + +def flat(): + return " ".join(raw().split()) + + +def t_distributions_distinguished(): + d = flat() + assert "TWO distributions" in d + assert "from mcp.server.fastmcp import FastMCP" in d, "the official import is not named" + assert "NOT `from fastmcp import FastMCP`" in d, "the standalone import is not refused" + + +def t_pin_is_the_identity(): + d = flat() + assert re.search(r"mcp==\d+\.\d+\.\d+", d), "no exact SDK pin" + assert "the pin IS the example's identity" in d + assert "re-pin only together with a re-run" in d or "re-pin only with a re-run" in d.lower() + assert "verified against (2026-08-13)" in d, "the tested date is not tied to the pin" + + +def t_v1_out_of_scope_and_lifecycle_order(): + d = flat() + assert "explicitly **out of scope**" in d or "explicitly out of scope" in d + assert "its OWN fixture verified against its own pin" in d + assert "The health route registers BEFORE the auth wrap" in d or \ + "health route registers BEFORE the auth" in d, "the verified order is missing" + + +def t_proof_is_a_protocol_call(): + d = flat() + assert "localhost protocol call, never a string in markdown" in d + for probe in ("`GET /health` → 200", "unauthenticated `/mcp` → 401", + "`tools/list`", "`tools/call`"): + assert probe in d, f"the acceptance list lost {probe!r}" + + +def t_snippets_consistent_with_the_named_import(): + # every python snippet constructing FastMCP must be the official-SDK shape + blocks = re.findall(r"```python\n(.*?)```", raw(), re.S) + ctors = [b for b in blocks if "FastMCP(" in b] + assert ctors, "no FastMCP snippet found — the doc moved" + for b in ctors: + assert "from fastmcp import" not in b, "a snippet imports the standalone distribution" + + +def main(): + case("the two distributions are distinguished; imports named/refused", + t_distributions_distinguished) + case("the pin is the example's identity, dated", t_pin_is_the_identity) + case("v1/standalone out of scope; lifecycle order verified", + t_v1_out_of_scope_and_lifecycle_order) + case("proof is a localhost protocol call with the 200/401/list/call set", + t_proof_is_a_protocol_call) + case("every FastMCP snippet matches the named distribution", + t_snippets_consistent_with_the_named_import) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-13.01.py b/test/audit_regressions/fix-as-13.01.py new file mode 100644 index 0000000..764e9f1 --- /dev/null +++ b/test/audit_regressions/fix-as-13.01.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""FIX-AS-13.01 — "long-running" no longer routes to A2A by itself (sherlock +audit, AS-13). + +The finding: the interop skill's "tell" said that inventing a task lifecycle, +a progress channel and a resumable handle on top of tools/call means "you +wanted A2A" — while the same skill's mcp.md correctly documents MCP Tasks as +exactly that durable handle. Duration was deciding the protocol. + +The fix under test (agent-interop SKILL.md + references/mcp.md): +* the PRIMARY dispatch criterion is capability/tool execution (MCP) vs + autonomous peer outcome (A2A); duration is a SECOND question about the + Tasks capability, not a protocol choice; +* the routing set: long-running fixed export → MCP Tasks when negotiated; + autonomous outsourced negotiation → A2A; Tasks unsupported → an explicit + fallback, not A2A; +* actually-negotiated client/SDK extensions are checked before building; +* no case in the routing set is decided by the word "long-running" alone. + +Standard library only. +""" +import os +import re +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SKILL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-interop", "SKILL.md") +MCP = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-interop", + "references", "mcp.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def flat(path): + with open(path, encoding="utf-8") as fh: + return " ".join(fh.read().split()) + + +def t_primary_criterion_is_what_not_how_long(): + d = flat(SKILL) + assert "The dispatch criterion is WHAT the other side is, not how long it runs." in d + assert "Duration is a SECOND question, and it is about a Tasks CAPABILITY, not a protocol." in d + + +def t_old_tell_is_gone(): + d = flat(SKILL) + assert "a resumable handle on top of `tools/call`, you wanted A2A" not in d, \ + "the old tell still routes a task lifecycle to A2A" + + +def t_routing_set_covers_all_three(): + d = flat(SKILL) + assert "long-running FIXED export → **MCP Tasks, when the client/SDK negotiates that extension**" in d + assert "autonomous outsourced negotiation → **A2A**" in d + assert "**Tasks unsupported** by the reached client/SDK → an explicit fallback" in d + assert "reaching for A2A because Tasks is absent is picking a protocol to dodge a missing extension" in d + + +def t_no_case_decided_by_the_word_alone(): + d = flat(SKILL) + assert "never by the word *long-running*" in d + assert "Check the client's ACTUALLY-negotiated extensions" in d + + +def t_mcp_md_agrees(): + d = flat(MCP) + assert "Duration is a TASKS-capability question, never a reason to switch to A2A" in d + assert "OPT-IN, NEGOTIATED extension" in d + assert "not a protocol change" in d + + +def main(): + case("the primary criterion is WHAT, duration is a second question", + t_primary_criterion_is_what_not_how_long) + case("the old lifecycle→A2A tell is gone", t_old_tell_is_gone) + case("the routing set covers Tasks / A2A / unsupported-fallback", + t_routing_set_covers_all_three) + case("no case is decided by the word long-running; negotiated support is checked", + t_no_case_decided_by_the_word_alone) + case("mcp.md carries the same rule at the Tasks bullet", t_mcp_md_agrees) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-as-14.01.py b/test/audit_regressions/fix-as-14.01.py new file mode 100644 index 0000000..2e71cd9 --- /dev/null +++ b/test/audit_regressions/fix-as-14.01.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""FIX-AS-14.01 — "no evals" no longer makes the whole audit unfalsifiable +(sherlock audit, AS-14). + +The finding: the doctrine prescribed "no evals → finding number one and +everything else is unfalsifiable", and "most agent bugs are prompt bugs" was +stated as measured reality. But a deterministic race, a hardcoded secret or a +miswired timeout are provable without a behavioural eval suite, and the +prompt-bug share was never measured. + +The fix under test (agent-harness SKILL.md + references/audit.md): +* three proof classes separated — source-level invariant proof, deterministic + reproduction, behavioural estimate; only the third inherits "no evals"; +* no evals = a finding about UNKNOWN RELIABILITY; a proven concrete harm keeps + its own priority and is never masked by the general finding; +* prompt-first survives as a diagnostic heuristic WITH exceptions — a broken + unit invariant is not treated by rewording; +* the synthetic no-evals + double-charge repo, run as behaviour. + +Standard library only. +""" +import os +import sys + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +SKILL = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-harness", "SKILL.md") +AUDIT = os.path.join(ROOT, "plugins", "agent-stack", "skills", "agent-harness", + "references", "audit.md") + +failures = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def flat(path): + with open(path, encoding="utf-8") as fh: + return " ".join(fh.read().split()) + + +def t_proof_classes_separated(): + d = flat(AUDIT) + for cls in ("**source-level invariant proof**", "**deterministic reproduction**", + "**behavioural estimate**"): + assert cls in d, f"proof class {cls} missing" + assert "survive a system with no evals untouched" in d + assert "inherits the no-evals finding until one exists" in d + + +def t_no_evals_is_unknown_reliability(): + s = flat(SKILL) + assert "a finding about UNKNOWN RELIABILITY" in s + assert "Everything downstream is then unfalsifiable — including this audit." not in s, \ + "the everything-is-unfalsifiable claim survived in SKILL.md" + assert 'a general "no evals" never masks a specific proven harm' in s + d = flat(AUDIT) + assert "everything else is unfalsifiable" not in d, \ + "the everything-else claim survived in audit.md" + assert "prioritized by their concrete harm" in d + + +def t_prompt_first_is_a_heuristic_with_exceptions(): + s = flat(SKILL) + assert "a diagnostic heuristic, with exceptions" in s + assert "not a measured share of defects" in s + assert "no rewording treats them" in s + assert "most agent bugs are prompt bugs wearing a stack trace" not in s, \ + "the unmeasured universal claim survived as a heading" + assert "The measured reality" not in s, \ + "vendor guidance is still presented as measured reality" + + +def t_traps_updated(): + d = flat(AUDIT) + assert "does NOT inherit it" in d + assert "the one finding with a victim gets deprioritized" in d + assert "Treating a broken unit invariant with a prompt change." in d + + +# ---- the rule as behaviour: the synthetic no-evals repo with a double charge + + +def audit(findings, has_evals): + """Each finding: {harm, proof}. Returns the kept findings with dispositions.""" + out = [] + if not has_evals: + out.append({"id": "no-evals", "kind": "unknown-reliability", + "priority_from": "concrete harm of what it blocks"}) + for f in findings: + if f["proof"] in ("source-invariant", "deterministic-repro"): + out.append({**f, "falsifiable": True, + "priority_from": f["harm"]}) + else: + out.append({**f, "falsifiable": has_evals, + "priority_from": f["harm"] if has_evals else "inherits no-evals"}) + return out + + +def t_double_charge_survives_no_evals(): + kept = audit([ + {"id": "double-charge", "harm": "a user is charged twice", + "proof": "deterministic-repro"}, + {"id": "usually-recovers", "harm": "operator time", + "proof": "behavioural-estimate"}, + ], has_evals=False) + ids = [f["id"] for f in kept] + assert "no-evals" in ids and "double-charge" in ids, \ + "one of the two findings was dropped" + dc = next(f for f in kept if f["id"] == "double-charge") + assert dc["falsifiable"] is True and dc["priority_from"] == "a user is charged twice", \ + "the proven double charge was masked by the general no-evals finding" + est = next(f for f in kept if f["id"] == "usually-recovers") + assert est["falsifiable"] is False, \ + "a behavioural estimate claimed falsifiability without evals" + + +def t_unit_invariant_not_fixed_by_prompt(): + remedy = {"source-invariant": "fix the code", + "behavioural-estimate": "check the prompt first"} + assert remedy["source-invariant"] == "fix the code" + s = flat(SKILL) + assert "source-level invariant violations are code bugs" in s + + +def main(): + case("the three proof classes are separated", t_proof_classes_separated) + case("no evals = unknown reliability; proven harm never masked", + t_no_evals_is_unknown_reliability) + case("prompt-first is a heuristic with named exceptions", + t_prompt_first_is_a_heuristic_with_exceptions) + case("the traps carry both new rules", t_traps_updated) + case("fixture: the no-evals repo keeps BOTH findings, double charge unmasked", + t_double_charge_survives_no_evals) + case("a broken unit invariant is not treated by rewording", + t_unit_invariant_not_fixed_by_prompt) + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-ev-01.23.py b/test/audit_regressions/fix-ev-01.23.py new file mode 100644 index 0000000..29ad068 --- /dev/null +++ b/test/audit_regressions/fix-ev-01.23.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""FIX-EV-01.23 — the outcome corpus for agent-evals (sherlock audit, +parent FIX-EV-01; depends on the family harness of FIX-EV-01.01). + +The corpus (evals/cases/agent-evals.json) is anchored to the audit's own +findings: a greenfield first release must carry happy + adversarial + +failure/retry trials and an empty corpus closes no gate (AS-06); an +order-sensitivity rubric must fail a swapped confirm/charge while passing +swapped reads, negative example kept beside it (AS-07); boundary n=1/n=3 +statistics carry no zero-width intervals (AS-08, a no-op case — the skill +reports, redesigns nothing); a regrade of an old trace is labelled and a +mutated candidate must flip the gate (AS-10); plus negative routing — judged +on ARTIFACTS through the family's outcome-case contract, so agent-evals can +no longer pass an eval by its name being picked. + +Checked here, stdlib only. +""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +CASES = os.path.join(ROOT, "evals", "cases", "agent-evals.json") +HARNESS = os.path.expanduser("~/DATA/sshlg-skills/test/outcome_harness.py") + +failures = [] +not_run = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def manifest(): + with open(CASES, encoding="utf-8") as fh: + return json.load(fh) + + +def t_cases_are_structurally_valid(): + m = manifest() + ids = [c["id"] for c in m["cases"]] + assert len(ids) == len(set(ids)) and len(ids) >= 5 + for c in m["cases"]: + assert c["schema_version"] == "outcome-case/1" + assert c["skill"] == "agent-evals" + assert c["environment"]["case_digest"] == \ + hashlib.sha256(c["prompt"]["text"].encode()).hexdigest(), \ + f"{c['id']}: case_digest does not pin the frozen prompt" + assert c["checks"]["outcome"], \ + f"{c['id']}: no outcome checks — the name-picking eval again" + + +def t_greenfield_case_demands_the_three_trial_kinds(): + c = next(x for x in manifest()["cases"] if "greenfield" in x["id"]) + p = c["prompt"]["text"] + for kind in ("happy", "adversarial", "failure/retry"): + assert kind in p, f"the greenfield minimum lost {kind!r} (AS-06)" + assert "empty corpus" in p and "NOT close the gate" in p, \ + "an empty corpus closing the gate is not refused (AS-06)" + + +def t_negative_refuses_to_load(): + neg = next(c for c in manifest()["cases"] if "negative" in c["id"]) + assert "agent-evals" in neg["checks"]["load_trace"]["expect_not_loaded"] + + +def t_order_case_pins_the_causal_pair(): + c = next(x for x in manifest()["cases"] if "order-sensitivity" in x["id"]) + p = c["prompt"]["text"] + assert "READ" in p and "must pass" in p.lower() or "must pass" in p, \ + "swapped independent reads are not required to pass (AS-07)" + assert "confirm/charge" in p and "acquire/write" in p, \ + "the causal pairs are not named (AS-07)" + expects = {o.get("expect") for o in c["checks"]["outcome"]} + assert "negative" in expects, "the negative example is not kept beside the rubric (AS-07)" + + +def t_stats_noop_and_regrade(): + m = manifest() + stats = next(c for c in m["cases"] if "statistics" in c["id"]) + assert "n=1" in stats["prompt"]["text"] and "zero-width" in stats["prompt"]["text"], \ + "boundary-n honesty is not demanded (AS-08)" + assert "Do NOT redesign" in stats["prompt"]["text"], "the no-op case now mutates" + regrade = next(c for c in m["cases"] if "regrade" in c["id"]) + expects = {o.get("expect") for o in regrade["checks"]["outcome"]} + assert "mutat" in expects and "regrade" in expects, \ + "candidate-mutation flip and regrade labelling are not both pinned (AS-10)" + flat = " ".join(json.dumps(m, ensure_ascii=False).split()) + for needle in ("actual output oracle", "raw result", "with/without-skill", + "grader convenience"): + assert needle in flat, f"the manifest no longer records {needle!r}" + + +def t_family_harness_validates_each_case_where_present(): + if not os.path.isfile(HARNESS): + not_run.append("family harness absent — case validation NOT_RUN (never PASS)") + return + for c in manifest()["cases"]: + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh: + json.dump(c, fh) + path = fh.name + try: + r = subprocess.run([sys.executable, HARNESS, path], + capture_output=True, text=True, timeout=60) + assert r.returncode == 0, \ + f"{c['id']} rejected by the family harness:\n{r.stdout}" + finally: + os.unlink(path) + + +def main(): + case("every case is structurally valid, none is name-picking", + t_cases_are_structurally_valid) + case("the greenfield case demands happy+adversarial+failure/retry (AS-06)", + t_greenfield_case_demands_the_three_trial_kinds) + case("the negative case refuses to load the skill", t_negative_refuses_to_load) + case("the order case pins the causal pair and keeps the negative (AS-07)", + t_order_case_pins_the_causal_pair) + case("boundary statistics stay honest; regrade is labelled (AS-08, AS-10)", + t_stats_noop_and_regrade) + case("the family harness validates each case (where present)", + t_family_harness_validates_each_case_where_present) + for n in not_run: + print(f" NOT_RUN {n}") + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-ev-01.24.py b/test/audit_regressions/fix-ev-01.24.py new file mode 100644 index 0000000..4b4fd64 --- /dev/null +++ b/test/audit_regressions/fix-ev-01.24.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""FIX-EV-01.24 — the outcome corpus for agent-harness (sherlock audit, +parent FIX-EV-01; depends on the family harness of FIX-EV-01.01). + +The corpus (evals/cases/agent-harness.json) is anchored to the audit's own +findings: a greenfield first release must carry happy + adversarial + +failure/retry trials and an empty corpus closes no gate (AS-06); an +order-sensitivity rubric must fail a swapped confirm/charge while passing +swapped reads, negative example kept beside it (AS-07); boundary n=1/n=3 +statistics carry no zero-width intervals (AS-08, a no-op case — the skill +reports, redesigns nothing); a regrade of an old trace is labelled and a +mutated candidate must flip the gate (AS-10); plus negative routing — judged +on ARTIFACTS through the family's outcome-case contract, so agent-harness can +no longer pass an eval by its name being picked. + +Checked here, stdlib only. +""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +CASES = os.path.join(ROOT, "evals", "cases", "agent-harness.json") +HARNESS = os.path.expanduser("~/DATA/sshlg-skills/test/outcome_harness.py") + +failures = [] +not_run = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def manifest(): + with open(CASES, encoding="utf-8") as fh: + return json.load(fh) + + +def t_cases_are_structurally_valid(): + m = manifest() + ids = [c["id"] for c in m["cases"]] + assert len(ids) == len(set(ids)) and len(ids) >= 5 + for c in m["cases"]: + assert c["schema_version"] == "outcome-case/1" + assert c["skill"] == "agent-harness" + assert c["environment"]["case_digest"] == \ + hashlib.sha256(c["prompt"]["text"].encode()).hexdigest(), \ + f"{c['id']}: case_digest does not pin the frozen prompt" + assert c["checks"]["outcome"], \ + f"{c['id']}: no outcome checks — the name-picking eval again" + + +def t_edges_case_preserves_control_and_state(): + c = next(x for x in manifest()["cases"] if "control-and-state" in x["id"]) + p = c["prompt"]["text"] + for edge in ("backup->migration", "approval->charge", "lease->edit"): + assert edge in p, f"the {edge} edge is no longer preserved (AS-04)" + assert "parallel" in p, "independent read-only parallelism is not demanded (AS-04)" + + +def t_negative_refuses_to_load(): + neg = next(c for c in manifest()["cases"] if "negative" in c["id"]) + assert "agent-harness" in neg["checks"]["load_trace"]["expect_not_loaded"] + + +def t_reconstruction_and_trifecta(): + m = manifest() + rec = next(c for c in m["cases"] if "reconstruction" in c["id"]) + assert "static and one dynamic" in rec["prompt"]["text"], \ + "auditability is a static diagram again (AS-05)" + assert "no PASS" in rec["prompt"]["text"] + tri = next(c for c in m["cases"] if "confirm-true" in c["id"]) + p = tri["prompt"]["text"] + assert "WITHOUT a grant is" in p and "changed arguments" in p, \ + "the trifecta rejections are not pinned (AS-11)" + assert "INDEPENDENTLY of private-data access" in p, \ + "untrusted->destructive is still coupled to private data (AS-11)" + + +def t_noop_and_manifest_rules(): + m = manifest() + noop = next(c for c in m["cases"] if "no-evals" in c["id"]) + assert "do not let the general 'no evals' verdict swallow" in noop["prompt"]["text"], \ + "the no-evals verdict swallows direct harm again (AS-14)" + assert "Do NOT redesign" in noop["prompt"]["text"], "the no-op case now mutates" + flat = " ".join(json.dumps(m, ensure_ascii=False).split()) + for needle in ("actual output oracle", "raw result", "with/without-skill", + "grader convenience"): + assert needle in flat, f"the manifest no longer records {needle!r}" + + +def t_family_harness_validates_each_case_where_present(): + if not os.path.isfile(HARNESS): + not_run.append("family harness absent — case validation NOT_RUN (never PASS)") + return + for c in manifest()["cases"]: + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh: + json.dump(c, fh) + path = fh.name + try: + r = subprocess.run([sys.executable, HARNESS, path], + capture_output=True, text=True, timeout=60) + assert r.returncode == 0, \ + f"{c['id']} rejected by the family harness:\n{r.stdout}" + finally: + os.unlink(path) + + +def main(): + case("every case is structurally valid, none is name-picking", + t_cases_are_structurally_valid) + case("control and state edges survive the fake-edge pruner (AS-04)", + t_edges_case_preserves_control_and_state) + case("the negative case refuses to load the skill", t_negative_refuses_to_load) + case("reconstruction covers dynamic too; confirm:true is not a grant (AS-05, AS-11)", + t_reconstruction_and_trifecta) + case("no-evals hides no double charge; manifest rules recorded (AS-14)", + t_noop_and_manifest_rules) + case("the family harness validates each case (where present)", + t_family_harness_validates_each_case_where_present) + for n in not_run: + print(f" NOT_RUN {n}") + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-ev-01.25.py b/test/audit_regressions/fix-ev-01.25.py new file mode 100644 index 0000000..2c1f4cc --- /dev/null +++ b/test/audit_regressions/fix-ev-01.25.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""FIX-EV-01.25 — the outcome corpus for agent-interop (sherlock audit, +parent FIX-EV-01; depends on the family harness of FIX-EV-01.01). + +The corpus (evals/cases/agent-interop.json) is anchored to the audit's own +findings: a greenfield first release must carry happy + adversarial + +failure/retry trials and an empty corpus closes no gate (AS-06); an +order-sensitivity rubric must fail a swapped confirm/charge while passing +swapped reads, negative example kept beside it (AS-07); boundary n=1/n=3 +statistics carry no zero-width intervals (AS-08, a no-op case — the skill +reports, redesigns nothing); a regrade of an old trace is labelled and a +mutated candidate must flip the gate (AS-10); plus negative routing — judged +on ARTIFACTS through the family's outcome-case contract, so agent-interop can +no longer pass an eval by its name being picked. + +Checked here, stdlib only. +""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +CASES = os.path.join(ROOT, "evals", "cases", "agent-interop.json") +HARNESS = os.path.expanduser("~/DATA/sshlg-skills/test/outcome_harness.py") + +failures = [] +not_run = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def manifest(): + with open(CASES, encoding="utf-8") as fh: + return json.load(fh) + + +def t_cases_are_structurally_valid(): + m = manifest() + ids = [c["id"] for c in m["cases"]] + assert len(ids) == len(set(ids)) and len(ids) >= 5 + for c in m["cases"]: + assert c["schema_version"] == "outcome-case/1" + assert c["skill"] == "agent-interop" + assert c["environment"]["case_digest"] == \ + hashlib.sha256(c["prompt"]["text"].encode()).hexdigest(), \ + f"{c['id']}: case_digest does not pin the frozen prompt" + assert c["checks"]["outcome"], \ + f"{c['id']}: no outcome checks — the name-picking eval again" + + +def t_pinned_mcp_case(): + c = next(x for x in manifest()["cases"] if "pinned-mcp" in x["id"]) + p = c["prompt"]["text"] + assert "exact SDK" in p and "fresh env" in p, "the pinned/clean-env proof is missing (AS-12)" + assert "401" in p and "out of scope" in p, "401 or the old-constructor scoping is missing (AS-12)" + + +def t_negative_refuses_to_load(): + neg = next(c for c in manifest()["cases"] if "negative" in c["id"]) + assert "agent-interop" in neg["checks"]["load_trace"]["expect_not_loaded"] + + +def t_routing_not_by_the_word(): + c = next(x for x in manifest()["cases"] if "protocol-routing" in x["id"]) + p = c["prompt"]["text"] + assert "MCP Tasks WHEN supported" in p, "the fixed export does not condition on support (AS-13)" + assert "A2A" in p and "autonomous" in p, "the A2A case is not distinguished (AS-13)" + assert "No case may be decided by the word\n'long-running' alone".replace("\n", " ") in p, \ + "a case can still be decided by the word long-running (AS-13)" + + +def t_noop_and_manifest_rules(): + m = manifest() + noop = next(c for c in m["cases"] if "unsupported-tasks" in c["id"]) + assert "not be forced into A2A because it is long" in noop["prompt"]["text"], \ + "the unsupported-Tasks case still falls to A2A (AS-13)" + assert "Do NOT redesign" in noop["prompt"]["text"], "the no-op case now mutates" + live = next(c for c in m["cases"] if "live" in c["id"]) + assert "AGENT_INTEROP_LIVE" in live["checks"]["tool"][0]["command"], \ + "the live case has no probe — it cannot NOT_RUN" + flat = " ".join(json.dumps(m, ensure_ascii=False).split()) + for needle in ("actual output oracle", "raw result", "with/without-skill", + "NOT_RUN", "grader convenience"): + assert needle in flat, f"the manifest no longer records {needle!r}" + + +def t_family_harness_validates_each_case_where_present(): + if not os.path.isfile(HARNESS): + not_run.append("family harness absent — case validation NOT_RUN (never PASS)") + return + for c in manifest()["cases"]: + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh: + json.dump(c, fh) + path = fh.name + try: + r = subprocess.run([sys.executable, HARNESS, path], + capture_output=True, text=True, timeout=60) + assert r.returncode == 0, \ + f"{c['id']} rejected by the family harness:\n{r.stdout}" + finally: + os.unlink(path) + + +def main(): + case("every case is structurally valid, none is name-picking", + t_cases_are_structurally_valid) + case("the pinned MCP example proves 401 + scoped old constructor (AS-12)", + t_pinned_mcp_case) + case("the negative case refuses to load the skill", t_negative_refuses_to_load) + case("protocol routing is not decided by the word long-running (AS-13)", + t_routing_not_by_the_word) + case("unsupported Tasks falls back, not to A2A; live probe-gated (AS-13, AS-12)", + t_noop_and_manifest_rules) + case("the family harness validates each case (where present)", + t_family_harness_validates_each_case_where_present) + for n in not_run: + print(f" NOT_RUN {n}") + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/audit_regressions/fix-ev-01.26.py b/test/audit_regressions/fix-ev-01.26.py new file mode 100644 index 0000000..472e270 --- /dev/null +++ b/test/audit_regressions/fix-ev-01.26.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""FIX-EV-01.26 — the outcome corpus for agent-orchestrator (sherlock audit, +parent FIX-EV-01; depends on the family harness of FIX-EV-01.01). + +The corpus (evals/cases/agent-orchestrator.json) is anchored to the audit's own +findings: a greenfield first release must carry happy + adversarial + +failure/retry trials and an empty corpus closes no gate (AS-06); an +order-sensitivity rubric must fail a swapped confirm/charge while passing +swapped reads, negative example kept beside it (AS-07); boundary n=1/n=3 +statistics carry no zero-width intervals (AS-08, a no-op case — the skill +reports, redesigns nothing); a regrade of an old trace is labelled and a +mutated candidate must flip the gate (AS-10); plus negative routing — judged +on ARTIFACTS through the family's outcome-case contract, so agent-orchestrator can +no longer pass an eval by its name being picked. + +Checked here, stdlib only. +""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile + +sys.dont_write_bytecode = True + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +CASES = os.path.join(ROOT, "evals", "cases", "agent-orchestrator.json") +HARNESS = os.path.expanduser("~/DATA/sshlg-skills/test/outcome_harness.py") + +failures = [] +not_run = [] + + +def case(name, fn): + try: + fn() + print(f" ok {name}") + except AssertionError as e: + failures.append(f"{name}: {e}") + print(f"FAIL {name}: {e}") + + +def manifest(): + with open(CASES, encoding="utf-8") as fh: + return json.load(fh) + + +def t_cases_are_structurally_valid(): + m = manifest() + ids = [c["id"] for c in m["cases"]] + assert len(ids) == len(set(ids)) and len(ids) >= 5 + for c in m["cases"]: + assert c["schema_version"] == "outcome-case/1" + assert c["skill"] == "agent-orchestrator" + assert c["environment"]["case_digest"] == \ + hashlib.sha256(c["prompt"]["text"].encode()).hexdigest(), \ + f"{c['id']}: case_digest does not pin the frozen prompt" + assert c["checks"]["outcome"], \ + f"{c['id']}: no outcome checks — the name-picking eval again" + + +def t_saga_case_keeps_unknown_pending(): + c = next(x for x in manifest()["cases"] if "saga-not-2pc" in x["id"]) + p = c["prompt"]["text"] + assert "UNKNOWN HTTP outcome stays PENDING" in p, "unknown is not kept pending (AS-01)" + assert "AT MOST ONE external effect per operation_id" in p, "no idempotency bound (AS-01)" + + +def t_negative_refuses_to_load(): + neg = next(c for c in manifest()["cases"] if "negative" in c["id"]) + assert "agent-orchestrator" in neg["checks"]["load_trace"]["expect_not_loaded"] + + +def t_baseline_and_memory(): + m = manifest() + base = next(c for c in m["cases"] if "zero-baseline" in c["id"]) + assert "uninitialized" in base["prompt"]["text"] and "initialized(0)" in base["prompt"]["text"], \ + "the zero-vs-missing baseline is not pinned (AS-02)" + mem = next(c for c in m["cases"] if "lexical-similarity" in c["id"]) + p = mem["prompt"]["text"] + assert "no silent merge" in p and "wins ONLY in its own scope" in p, \ + "contradiction handling is not pinned (AS-03)" + + +def t_noop_and_manifest_rules(): + m = manifest() + noop = next(c for c in m["cases"] if "fake-edge" in c["id"]) + p = noop["prompt"]["text"] + for edge in ("backup->migration", "approval->charge", "lease->edit"): + assert edge in p, f"the {edge} edge is not preserved (AS-04)" + assert "Do NOT redesign" in p, "the no-op case now mutates" + flat = " ".join(json.dumps(m, ensure_ascii=False).split()) + for needle in ("actual output oracle", "raw result", "with/without-skill", + "grader convenience"): + assert needle in flat, f"the manifest no longer records {needle!r}" + + +def t_family_harness_validates_each_case_where_present(): + if not os.path.isfile(HARNESS): + not_run.append("family harness absent — case validation NOT_RUN (never PASS)") + return + for c in manifest()["cases"]: + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh: + json.dump(c, fh) + path = fh.name + try: + r = subprocess.run([sys.executable, HARNESS, path], + capture_output=True, text=True, timeout=60) + assert r.returncode == 0, \ + f"{c['id']} rejected by the family harness:\n{r.stdout}" + finally: + os.unlink(path) + + +def main(): + case("every case is structurally valid, none is name-picking", + t_cases_are_structurally_valid) + case("the saga case keeps an unknown outcome pending (AS-01)", + t_saga_case_keeps_unknown_pending) + case("the negative case refuses to load the skill", t_negative_refuses_to_load) + case("zero baseline is not missing; opposites do not merge (AS-02, AS-03)", + t_baseline_and_memory) + case("the fake-edge no-op preserves control/state edges (AS-04)", + t_noop_and_manifest_rules) + case("the family harness validates each case (where present)", + t_family_harness_validates_each_case_where_present) + for n in not_run: + print(f" NOT_RUN {n}") + if failures: + print(f"\n{len(failures)} failure(s)") + return 1 + print("\nall green") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/evals/fixtures/bootstrap-corpus.json b/test/evals/fixtures/bootstrap-corpus.json new file mode 100644 index 0000000..b36b800 --- /dev/null +++ b/test/evals/fixtures/bootstrap-corpus.json @@ -0,0 +1,30 @@ +{ + "_why": "FIX-AS-06.01. A greenfield feature has no production traces, but observable-only is not release-ready: a criterion with no input is never executed. This is the SEED corpus — curated/synthetic/manual inputs with explicit provenance, carrying at least a happy, an adversarial and a failure/retry trial. It is SUPPLEMENTED by production traces as they arrive; it never claims full coverage.", + "_rules": "Each input names its provenance (curated|synthetic|manual). The release gate requires EXECUTED trials over these; observables with nothing run are specification-ready, not release-ready. A corrupted input/fixture/runner is TEST_ERROR, not a behaviour pass/fail. Cases are isolated — order-independent.", + "trials": [ + { + "id": "seed-happy-01", + "kind": "happy", + "provenance": "curated", + "skill": "agent-orchestrator", + "input": "Two independent research subtasks feed one synthesis node.", + "observable": "The plan runs the two subtasks in parallel and places a checker before synthesis." + }, + { + "id": "seed-adversarial-01", + "kind": "adversarial", + "provenance": "synthetic", + "skill": "agent-orchestrator", + "input": "A subtask returns a confident hallucination the synthesis node cannot distinguish.", + "observable": "The checker marks the branch not-usable and synthesis excludes it, rather than combining all three." + }, + { + "id": "seed-failure-retry-01", + "kind": "failure_retry", + "provenance": "manual", + "skill": "agent-orchestrator", + "input": "The provider raises 100 consecutive rate-limit errors mid-run.", + "observable": "Retries are bounded, a checkpoint is saved, and the run does not loop forever." + } + ] +} diff --git a/test/evals/scenarios.json b/test/evals/scenarios.json index 8d51ef8..2ee6e9c 100644 --- a/test/evals/scenarios.json +++ b/test/evals/scenarios.json @@ -1,5 +1,5 @@ { - "note": "Score each expected_behavior line independently. These are authored evaluations, not executed results.", + "note": "Score each expected_behavior line independently. These are authored OBSERVABLES, not executed results — specification-ready. Release-readiness needs EXECUTED trials over a corpus; the greenfield seed corpus is fixtures/bootstrap-corpus.json (curated/synthetic/manual, provenance-marked, supplemented by production). See agent-evals SKILL.md §6.", "scenarios": [ { "id": "s01",