From afc1403207b6d01d436c41b2973a55b3e0eccbdc Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:42:27 +0900 Subject: [PATCH 1/7] fix(review): judge test-integrity determinism by seed ids across every repeat Issue #28 requires three repeated reviews to detect the same seed IDs and says wording is not part of determinism. score compared whole predicted class sets, so a varying extra false alarm failed determinism, and it scored only the first repeat, so a clean-fixture P0/P1 false positive in a later repeat was ignored. Determinism now compares, for each seeded case, whether its own class was detected. Recall, precision, and clean-fixture false positives are computed per repeat; the gate and the headline figures use the worst repeat, and the report lists each repeat. Constraint: extra findings must still count against precision in each repeat Rejected: keep class-set equality | stricter than the issue and hides which repeat misses a seed Confidence: high Scope-risk: narrow Tested: scripts/test-eval-test-integrity-model.sh passes; reverting each of seed-id comparison, worst-repeat false positives, and worst-repeat precision makes it fail Not-tested: validate.sh not yet run on this commit; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- scripts/eval-test-integrity-model.py | 123 +++++++++++------- scripts/test-eval-test-integrity-model.sh | 37 ++++-- skills/reviewing/references/test-integrity.md | 2 +- 3 files changed, 106 insertions(+), 56 deletions(-) diff --git a/scripts/eval-test-integrity-model.py b/scripts/eval-test-integrity-model.py index c34a025..afda1f6 100755 --- a/scripts/eval-test-integrity-model.py +++ b/scripts/eval-test-integrity-model.py @@ -193,57 +193,88 @@ def score(args): else: batch[case["id"]] = findings predictions.append(batch) + # Determinism is judged on detected seed ids (issue #28): wording and extra + # findings do not count here. Extra findings still lower each repeat's own + # precision and false-positive counts below. repeat_mismatch = [] if args.repeat > 1: for case in cases: - sets = [None if batch[case["id"]] is None else sorted(set(f["integrity_class"] for f in batch[case["id"]])) for batch in predictions] - if any(item != sets[0] for item in sets[1:]): + if not case["expected_findings"]: + continue + detected = [None if batch[case["id"]] is None else any(f["integrity_class"] == case["class"] for f in batch[case["id"]]) for batch in predictions] + if any(item != detected[0] for item in detected[1:]): repeat_mismatch.append(case["id"]) - tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0 - per_class = {} - for class_id, class_severity in sorted(classes.items()): - per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0} - for case in cases: - expected = bool(case["expected_findings"]) - class_id = case["class"] - class_severity = classes.get(class_id) - if expected: - expected_positive += 1 - per_class.setdefault(class_id, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["expected"] += 1 - if class_severity in ("P0", "P1"): - p01_expected += 1 - findings = predictions[0][case["id"]] - if findings is None: - details.append({"case_id": case["id"], "status": "malformed-or-missing"}) - continue - predicted = set(f["integrity_class"] for f in findings) - predicted_count += len(findings) - if expected and class_id in predicted: - tp += 1 - per_class[class_id]["true_positive"] += 1 - if class_severity in ("P0", "P1"): - p01_tp += 1 - if any(f["integrity_class"] == class_id and f["severity"] == class_severity for f in findings): - severity_match += 1 - per_class[class_id]["severity_agreement"] += 1 - for f in findings: - pred_class = f["integrity_class"] - correct = expected and pred_class == class_id - if not correct: - fp += 1 - per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1 - if not expected and f["severity"] in ("P0", "P1"): - p01_negative_fp += 1 - if pred_class not in classes: - # Unknown class identifiers are malformed schema values. - malformed.append({"case_id": case["id"], "reason": "unknown class " + pred_class}) - details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)}) - precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0) - recall = float(tp) / expected_positive if expected_positive else 1.0 - p01_recall = float(p01_tp) / p01_expected if p01_expected else 1.0 - report = {"positive_recall": recall, "p0_p1_recall": p01_recall, "precision": precision, "clean_negative_p0_p1_false_positives": p01_negative_fp, "malformed_responses": malformed, "repeat_mismatches": repeat_mismatch, "severity_agreement_count": severity_match, "predicted_finding_count": predicted_count, "true_positive_count": tp, "false_positive_count": fp, "per_class": per_class, "cases": details} + + def evaluate(batch): + tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0 + per_class = {} + details = [] + extra_malformed = [] + for class_id in sorted(classes): + per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0} + for case in cases: + expected = bool(case["expected_findings"]) + class_id = case["class"] + class_severity = classes.get(class_id) + if expected: + expected_positive += 1 + per_class.setdefault(class_id, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["expected"] += 1 + if class_severity in ("P0", "P1"): + p01_expected += 1 + findings = batch[case["id"]] + if findings is None: + details.append({"case_id": case["id"], "status": "malformed-or-missing"}) + continue + predicted = set(f["integrity_class"] for f in findings) + predicted_count += len(findings) + if expected and class_id in predicted: + tp += 1 + per_class[class_id]["true_positive"] += 1 + if class_severity in ("P0", "P1"): + p01_tp += 1 + if any(f["integrity_class"] == class_id and f["severity"] == class_severity for f in findings): + severity_match += 1 + per_class[class_id]["severity_agreement"] += 1 + for f in findings: + pred_class = f["integrity_class"] + correct = expected and pred_class == class_id + if not correct: + fp += 1 + per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1 + if not expected and f["severity"] in ("P0", "P1"): + p01_negative_fp += 1 + if pred_class not in classes: + # Unknown class identifiers are malformed schema values. + extra_malformed.append({"case_id": case["id"], "reason": "unknown class " + pred_class}) + details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)}) + precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0) + return { + "positive_recall": float(tp) / expected_positive if expected_positive else 1.0, + "p0_p1_recall": float(p01_tp) / p01_expected if p01_expected else 1.0, + "precision": precision, + "clean_negative_p0_p1_false_positives": p01_negative_fp, + "severity_agreement_count": severity_match, + "predicted_finding_count": predicted_count, + "true_positive_count": tp, + "false_positive_count": fp, + "per_class": per_class, + "cases": details, + "malformed_extra": extra_malformed, + } + + repeats = [evaluate(batch) for batch in predictions] + for item in repeats: + malformed.extend(item.pop("malformed_extra")) + # The gate must hold in every repeat; the headline figures are the worst repeat. + worst = dict(repeats[0]) + worst["positive_recall"] = min(item["positive_recall"] for item in repeats) + worst["p0_p1_recall"] = min(item["p0_p1_recall"] for item in repeats) + worst["precision"] = min(item["precision"] for item in repeats) + worst["clean_negative_p0_p1_false_positives"] = max(item["clean_negative_p0_p1_false_positives"] for item in repeats) + report = dict(worst, malformed_responses=malformed, repeat_mismatches=repeat_mismatch) + report["repeats"] = [{key: item[key] for key in ("positive_recall", "p0_p1_recall", "precision", "clean_negative_p0_p1_false_positives", "false_positive_count", "true_positive_count")} for item in repeats] print(json.dumps(report, indent=2, sort_keys=True)) - if p01_recall < 1.0 or precision < 0.90 or p01_negative_fp or malformed or repeat_mismatch: + if report["p0_p1_recall"] < 1.0 or report["precision"] < 0.90 or report["clean_negative_p0_p1_false_positives"] or malformed or repeat_mismatch: raise SystemExit(1) diff --git a/scripts/test-eval-test-integrity-model.sh b/scripts/test-eval-test-integrity-model.sh index daed51a..8164fa8 100755 --- a/scripts/test-eval-test-integrity-model.sh +++ b/scripts/test-eval-test-integrity-model.sh @@ -152,15 +152,34 @@ with tempfile.TemporaryDirectory(prefix="c2ti-model-test-") as temporary: cross_report = json.loads(cross_result.stdout) assert cross_report["false_positive_count"] == 1 - repeat_a = base / "repeat-a" - repeat_b = base / "repeat-b" - repeat_a.mkdir(); repeat_b.mkdir() - for directory in (repeat_a, repeat_b): - (directory / "CASE-SECRET-CLEAN.json").write_text('{"findings": []}', encoding="utf-8") - (repeat_a / "CASE-SECRET-1.json").write_text('{"findings": []}', encoding="utf-8") - (repeat_b / "CASE-SECRET-1.json").write_text(json.dumps({"findings": [{"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "network call"}]}), encoding="utf-8") - repeat_result = call(["score", "--plan", str(plan_path), "--responses", str(repeat_a), str(repeat_b), "--corpus", str(corpus), "--repeat", "2"], expected=1) - assert "repeat_mismatches" in repeat_result.stdout + # Determinism compares detected seed ids, as issue #28 requires; every + # repeat is also scored on its own. + def write_repeat(name, seed_findings, clean_findings): + directory = base / name + directory.mkdir() + (directory / "CASE-SECRET-1.json").write_text(json.dumps({"findings": seed_findings}), encoding="utf-8") + (directory / "CASE-SECRET-CLEAN.json").write_text(json.dumps({"findings": clean_findings}), encoding="utf-8") + return directory + hit = {"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "network call"} + noise = {"integrity_class": "assertion-specificity", "severity": "P2", "line": 1, "evidence": "noise"} + clean_fp = {"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "false alarm"} + # A seed detected in one repeat and missed in another is a mismatch. + flip = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("flip-a", [hit], [])), str(write_repeat("flip-b", [], [])), "--corpus", str(corpus), "--repeat", "2"], expected=1) + assert json.loads(flip.stdout)["repeat_mismatches"] == ["CASE-SECRET-1"], flip.stdout + # An extra non-seed class varying across repeats is not a seed mismatch; + # it is counted against that repeat's precision instead. + extra_class = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("noise-a", [hit], [])), str(write_repeat("noise-b", [hit, noise], [])), "--corpus", str(corpus), "--repeat", "2"], expected=1) + extra_report = json.loads(extra_class.stdout) + assert extra_report["repeat_mismatches"] == [], extra_class.stdout + assert [item["precision"] for item in extra_report["repeats"]] == [1.0, 0.5], extra_class.stdout + # A clean-case P0/P1 false positive in a later repeat fails the gate. + late_fp = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("late-a", [hit], [])), str(write_repeat("late-b", [hit], [clean_fp])), "--corpus", str(corpus), "--repeat", "2"], expected=1) + late_report = json.loads(late_fp.stdout) + assert late_report["repeat_mismatches"] == [], late_fp.stdout + assert [item["clean_negative_p0_p1_false_positives"] for item in late_report["repeats"]] == [0, 1], late_fp.stdout + assert late_report["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout + # Identical passing repeats pass. + call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("same-a", [hit], [])), str(write_repeat("same-b", [hit], [])), "--corpus", str(corpus), "--repeat", "2"]) # The same directory cannot stand in for several repeats. call(["score", "--plan", str(plan_path), "--responses", str(good), str(good), "--corpus", str(corpus), "--repeat", "2"], expected=1) diff --git a/skills/reviewing/references/test-integrity.md b/skills/reviewing/references/test-integrity.md index b567d53..c8f51a9 100644 --- a/skills/reviewing/references/test-integrity.md +++ b/skills/reviewing/references/test-integrity.md @@ -103,4 +103,4 @@ python3 scripts/eval-test-integrity-model.py run \ --approval ``` -For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all four `--corpus` arguments to `score` as in `plan`. The gate requires 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, no malformed or missing responses, and identical predicted class-id sets in every repeat. The report also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. +For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all four `--corpus` arguments to `score` as in `plan`. The gate requires, in every repeat, 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, and no malformed or missing responses. Determinism requires the same set of detected seed ids in every repeat; free-text wording and extra findings do not count toward it, but extra findings still lower each repeat's precision. The report shows the worst repeat as its headline figures, lists each repeat under `repeats`, and also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. From 220b733bd243a7d7d820ea260fadcb2d06d0a18d Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:12:58 +0900 Subject: [PATCH 2/7] feat(review): tighten test-integrity class boundaries and add a blind gate-2 corpus A measured run of the model gate (3 repeats, 79 cases) showed three systematic errors: unrelated Assertion specificity findings on fragments whose defect belongs to another class, fully local Git fixtures flagged as Fixture Git signing/hooks, and unflagged HOME/PATH reads. The checklist now states each boundary. Those four corpora were used to tune the wording, so they no longer measure out-of-sample performance. test-integrity-blind-gate2.json adds 30 cases (3 defective and 2 clean per class) written by an author who saw only class descriptions; two independent judges labeling from the sources alone agreed with all 30 labels. Constraint: tuned corpora must not be reported as out-of-sample evidence Rejected: change the 90% precision or 100% recall thresholds | the gate belongs to the issue, not to this change Confidence: medium Scope-risk: narrow Tested: scripts/test-eval-test-integrity-model.sh and bash scripts/validate.sh passed before the corpus and doc edit; the new corpus parses with plan Not-tested: model behavior under the new wording; validate.sh on this exact commit; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- skills/reviewing/references/test-integrity.md | 10 +- .../review/test-integrity-blind-gate2.json | 336 ++++++++++++++++++ 2 files changed, 342 insertions(+), 4 deletions(-) create mode 100644 tests/conformance/review/test-integrity-blind-gate2.json diff --git a/skills/reviewing/references/test-integrity.md b/skills/reviewing/references/test-integrity.md index c8f51a9..c051894 100644 --- a/skills/reviewing/references/test-integrity.md +++ b/skills/reviewing/references/test-integrity.md @@ -38,13 +38,13 @@ Evidence cites the assertion and the untested distinction. Do not flag a deliber Check whether tests read host `HOME`, `PATH`, `TMPDIR`, locale, timezone, credentials, Git configuration outside fixture repositories, network proxy variables, or tool-specific configuration without setting or clearing them in the fixture. Flag environment reads that change the verdict or target and are not represented in the fixture's declared inputs. Severity is P1 when host state can change authorization or target selection and P2 otherwise. -Evidence names the environment key or implicit process state and the missing fixture declaration. Do not flag an explicitly supplied value that the fixture records and verifies. +Evidence names the environment key or implicit process state and the missing fixture declaration. A read of host `HOME`, `PATH`, or `TMPDIR` that the snippet does not first set or clear is a finding even when the snippet shown only stores or prints the value, because the fixture has not declared the input. Do not flag an explicitly supplied value that the fixture records and verifies. ### Fixture Git signing/hooks Check every temporary Git repository for explicit identity, signing policy, hooks path, and remote configuration. A fixture must not inherit host signing keys, global hooks, credential helpers, or remotes. Flag missing `commit.gpgsign=false` when the fixture must commit without host keys, missing empty `core.hooksPath`, and use of an inherited global configuration. Severity is P1 when host code can execute or a commit cannot be reproduced; otherwise P2. -Evidence cites the Git setup and the inherited state that remains possible. Do not flag a fixture that sets all required keys explicitly and proves its hooks directory is empty. +Evidence cites the Git setup and the inherited state that remains possible. Do not flag a fixture that sets identity, `commit.gpgsign=false`, and `core.hooksPath` explicitly with `--local` and proves its hooks directory is empty; setting those keys locally is complete isolation for this class, and a missing `GIT_CONFIG_GLOBAL` or `GIT_CONFIG_NOSYSTEM` is not a separate finding then. ### Class boundaries @@ -55,6 +55,7 @@ Report each defect once, under the single class that owns it: - An ignored exit status, swallowed exception, or default-success value belongs to Verifier fail-open behavior. Report Assertion specificity only for a test assertion that can pass without the required behavior. - Hidden environment dependencies covers host state other than Git configuration that the snippet reads or that selects its target or verdict. Running a tool or `git` binary from `PATH` is not by itself an environment defect. - Judge only the text shown. Do not report a class because unseen helper code might be unsafe. +- A snippet is often a fragment written to show one defect. When another class owns that defect, do not also report Assertion specificity because the fragment lacks an assertion; report that class only for an assertion that is present and too weak. ## Per-run evidence record @@ -76,7 +77,7 @@ Run `bash scripts/test-review-test-integrity.sh seeded` to exercise seed and hol ## Model release gate -The release gate scores the real model reviewer against four corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, and `tests/conformance/review/test-integrity-blind-final.json`. The blind corpora were authored without access to the detectors. The final corpus was written after the class boundaries were fixed and was not used to change the checklist, so it is the out-of-sample measurement; two independent judges agreed with all of its labels. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. +The release gate scores the real model reviewer against five corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, `tests/conformance/review/test-integrity-blind-final.json`, and `tests/conformance/review/test-integrity-blind-gate2.json`. The blind corpora were authored without access to the detectors. The first four were used to tune the checklist wording, so after that tuning they are regression data, not out-of-sample evidence. `test-integrity-blind-gate2.json` was written by an author who had only the class descriptions, before the latest checklist edit that responded to the measured errors; two independent judges, labeling from the sources alone, agreed with all 30 of its labels. Report its results separately as the out-of-sample measurement. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. First create the exact request plan without invoking a model: @@ -86,6 +87,7 @@ python3 scripts/eval-test-integrity-model.py plan \ --corpus tests/conformance/review/test-integrity-blind-dev.json \ --corpus tests/conformance/review/test-integrity-blind-confirm.json \ --corpus tests/conformance/review/test-integrity-blind-final.json \ + --corpus tests/conformance/review/test-integrity-blind-gate2.json \ --out /tmp/c2ti-model-plan shasum -a 256 /tmp/c2ti-model-plan/plan.json ``` @@ -103,4 +105,4 @@ python3 scripts/eval-test-integrity-model.py run \ --approval ``` -For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all four `--corpus` arguments to `score` as in `plan`. The gate requires, in every repeat, 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, and no malformed or missing responses. Determinism requires the same set of detected seed ids in every repeat; free-text wording and extra findings do not count toward it, but extra findings still lower each repeat's precision. The report shows the worst repeat as its headline figures, lists each repeat under `repeats`, and also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. +For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all five `--corpus` arguments to `score` as in `plan`. The gate requires, in every repeat, 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, and no malformed or missing responses. Determinism requires the same set of detected seed ids in every repeat; free-text wording and extra findings do not count toward it, but extra findings still lower each repeat's precision. The report shows the worst repeat as its headline figures, lists each repeat under `repeats`, and also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. diff --git a/tests/conformance/review/test-integrity-blind-gate2.json b/tests/conformance/review/test-integrity-blind-gate2.json new file mode 100644 index 0000000..bf1723a --- /dev/null +++ b/tests/conformance/review/test-integrity-blind-gate2.json @@ -0,0 +1,336 @@ +{ + "schema": "c2-ti/v1", + "description": "Gate-2 blind out-of-sample cases, authored without the checklist", + "cases": [ + { + "id": "C2-TI-G2-NET-P1", + "class": "network-isolation", + "kind": "holdout", + "file": "test_forecast_parse.py", + "source": "import requests\n\nfrom weather.client import parse_forecast\n\n\ndef test_parse_forecast_from_service_response():\n resp = requests.get(\n 'https://api.open-meteo.com/v1/forecast',\n params={'latitude': 52.52, 'longitude': 13.41, 'hourly': 'temperature_2m'},\n timeout=10,\n )\n resp.raise_for_status()\n payload = resp.json()\n forecast = parse_forecast(payload)\n assert forecast.latitude == payload['latitude']\n assert [t for t, _ in forecast.hourly] == payload['hourly']['time']\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-NET-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-NET-P2", + "class": "network-isolation", + "kind": "holdout", + "file": "test_depcheck_version.py", + "source": "import subprocess\nimport sys\n\nfrom depcheck import installed_version\n\n\ndef test_installed_version_reads_dist_metadata(tmp_path):\n target = tmp_path / 'site'\n subprocess.run(\n [sys.executable, '-m', 'pip', 'install', '--quiet', '--target', str(target), 'tomli==2.0.1'],\n check=True,\n )\n assert installed_version('tomli', search_path=[str(target)]) == '2.0.1'\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-NET-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-NET-P3", + "class": "network-isolation", + "kind": "holdout", + "file": "test_tcp_banner.py", + "source": "import re\n\nfrom netprobe import tcp_banner\n\n\ndef test_tcp_banner_reads_smtp_greeting():\n banner = tcp_banner('smtp.gmail.com', 587, timeout=5.0)\n assert re.match(r'^220 \\S+ ESMTP', banner)\n assert not banner.endswith('\\r\\n')\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-NET-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-NET-N1", + "class": "network-isolation", + "kind": "holdout", + "file": "test_weather_client.py", + "source": "import httpx\n\nfrom weather.client import WeatherClient\n\n\ndef test_client_parses_hourly_series():\n def handler(request: httpx.Request) -> httpx.Response:\n assert request.url.path == '/v1/forecast'\n return httpx.Response(\n 200,\n json={'latitude': 52.5, 'hourly': {'time': ['t0', 't1'], 'temperature_2m': [3.0, 4.5]}},\n )\n\n transport = httpx.MockTransport(handler)\n with httpx.Client(transport=transport, base_url='https://api.open-meteo.com') as http:\n forecast = WeatherClient(http).forecast(52.5, 13.4)\n assert forecast.latitude == 52.5\n assert forecast.hourly == [('t0', 3.0), ('t1', 4.5)]\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-NET-N2", + "class": "network-isolation", + "kind": "holdout", + "file": "test_tcp_banner_loopback.py", + "source": "import socket\nimport threading\n\nfrom netprobe import tcp_banner\n\n\ndef test_tcp_banner_reads_first_line_from_loopback():\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv:\n srv.bind(('127.0.0.1', 0))\n srv.listen(1)\n port = srv.getsockname()[1]\n\n def serve():\n conn, _ = srv.accept()\n with conn:\n conn.sendall(b'220 mail.test ESMTP ready\\r\\n')\n\n worker = threading.Thread(target=serve)\n worker.start()\n banner = tcp_banner('127.0.0.1', port, timeout=5.0)\n worker.join(timeout=5)\n assert banner == '220 mail.test ESMTP ready'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-TMP-P1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "test_export_report.py", + "source": "import json\nimport os\n\nfrom exporter import export_report\n\n\ndef test_export_report_writes_json():\n out = '/tmp/report-export.json'\n if os.path.exists(out):\n os.remove(out)\n export_report({'total': 3, 'failed': 1}, out)\n with open(out) as fh:\n data = json.load(fh)\n assert data == {'total': 3, 'failed': 1}\n os.remove(out)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-TMP-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-TMP-P2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "test_thumbnails.py", + "source": "import shutil\nimport tempfile\nfrom pathlib import Path\n\nfrom thumbs import render_thumbnails\n\n\ndef test_render_thumbnails_creates_one_file_per_image(sample_images):\n outdir = Path(tempfile.gettempdir()) / 'thumbs-out'\n shutil.rmtree(outdir, ignore_errors=True)\n outdir.mkdir()\n render_thumbnails(sample_images, outdir, size=(64, 64))\n produced = sorted(p.name for p in outdir.iterdir())\n assert produced == sorted(f'{img.stem}.png' for img in sample_images)\n shutil.rmtree(outdir)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-TMP-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-TMP-P3", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "csvsplit_test.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d /tmp/csvsplit.XXXXXX)\ntrap 'rm -rf /tmp/csvsplit.*' EXIT\nprintf 'id,name\\n1,a\\n2,b\\n3,c\\n' > \"$work/in.csv\"\n./bin/csvsplit --rows 2 --out \"$work\" \"$work/in.csv\"\n[ \"$(cat \"$work/part-1.csv\")\" = \"$(printf 'id,name\\n1,a\\n2,b')\" ] || { echo \"part-1 mismatch\" >&2; exit 1; }\n[ \"$(cat \"$work/part-2.csv\")\" = \"$(printf 'id,name\\n3,c')\" ] || { echo \"part-2 mismatch\" >&2; exit 1; }\necho ok\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-TMP-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-TMP-N1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "test_export_roundtrip.py", + "source": "import json\nimport tempfile\nfrom pathlib import Path\n\nfrom exporter import export_report\n\n\ndef test_export_report_round_trips():\n with tempfile.TemporaryDirectory(prefix='report-') as d:\n out = Path(d) / 'report.json'\n export_report({'total': 3, 'failed': 1}, str(out))\n assert json.loads(out.read_text()) == {'total': 3, 'failed': 1}\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-TMP-N2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "csvsplit_parts.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ncleanup() { rm -rf \"$work\"; }\ntrap cleanup EXIT INT TERM\nprintf 'id,name\\n1,a\\n2,b\\n3,c\\n' > \"$work/in.csv\"\n./bin/csvsplit --rows 2 --out \"$work/parts\" \"$work/in.csv\"\n[ \"$(cat \"$work/parts/part-1.csv\")\" = \"$(printf 'id,name\\n1,a\\n2,b')\" ] || { echo \"part-1 mismatch\" >&2; exit 1; }\n[ \"$(cat \"$work/parts/part-2.csv\")\" = \"$(printf 'id,name\\n3,c')\" ] || { echo \"part-2 mismatch\" >&2; exit 1; }\n[ ! -e \"$work/parts/part-3.csv\" ] || { echo \"unexpected part-3\" >&2; exit 1; }\necho ok\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-VER-P1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "verify_schema.py", + "source": "import json\nimport subprocess\nimport sys\n\n\ndef verify(artifact_dir: str) -> dict:\n try:\n proc = subprocess.run(\n [sys.executable, '-m', 'schema_check', artifact_dir],\n capture_output=True, text=True, timeout=120,\n )\n report = json.loads(proc.stdout)\n except Exception:\n return {'verdict': 'pass', 'violations': []}\n violations = report.get('violations', [])\n return {'verdict': 'fail' if violations else 'pass', 'violations': violations}\n\n\nif __name__ == '__main__':\n result = verify(sys.argv[1])\n print(json.dumps(result))\n sys.exit(0 if result['verdict'] == 'pass' else 1)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-VER-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-VER-P2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "check_lint.sh", + "source": "#!/bin/sh\nreport=$(mktemp)\ntrap 'rm -f \"$report\"' EXIT\n./node_modules/.bin/eslint --format json src > \"$report\" 2>/dev/null\nerrors=$(python3 -c 'import json, sys; print(sum(f[\"errorCount\"] for f in json.load(open(sys.argv[1]))))' \"$report\" 2>/dev/null)\nif [ \"${errors:-0}\" -gt 0 ]; then\n echo \"lint: $errors error(s)\"\n exit 1\nfi\necho \"lint: clean\"\nexit 0\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-VER-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-VER-P3", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "grader_adapter.py", + "source": "from dataclasses import dataclass\n\n\n@dataclass\nclass Outcome:\n passed: bool\n detail: str\n\n\n_FAILING = {'fail', 'error'}\n\n\ndef adapt(raw: dict) -> Outcome:\n status = str(raw.get('status', '')).lower()\n if status in _FAILING:\n return Outcome(False, raw.get('message', status))\n return Outcome(True, raw.get('message', 'ok'))\n\n\ndef assert_grader_passed(raw: dict) -> None:\n outcome = adapt(raw)\n assert outcome.passed, outcome.detail\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-VER-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-VER-N1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "verify_schema_strict.py", + "source": "import json\nimport subprocess\nimport sys\n\ndef verify(artifact_dir: str) -> dict:\n try:\n proc = subprocess.run(\n [sys.executable, '-m', 'schema_check', artifact_dir],\n capture_output=True, text=True, timeout=120,\n )\n except subprocess.TimeoutExpired:\n return {'verdict': 'error', 'reason': 'timeout'}\n if proc.returncode not in (0, 1):\n return {'verdict': 'error', 'reason': f'exit {proc.returncode}'}\n try:\n violations = json.loads(proc.stdout)['violations']\n except (json.JSONDecodeError, KeyError, TypeError):\n return {'verdict': 'error', 'reason': 'unparseable report'}\n verdict = 'pass' if violations == [] and proc.returncode == 0 else 'fail'\n return {'verdict': verdict, 'violations': violations}\n\nif __name__ == '__main__':\n result = verify(sys.argv[1])\n print(json.dumps(result))\n sys.exit(0 if result['verdict'] == 'pass' else 1)\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-VER-N2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "grader_outcome.py", + "source": "from dataclasses import dataclass\n\n\n@dataclass\nclass Outcome:\n passed: bool\n detail: str\n\n\ndef adapt(raw: dict) -> Outcome:\n status = raw.get('status')\n if status == 'pass':\n return Outcome(True, raw.get('message', 'pass'))\n if status in ('fail', 'error'):\n return Outcome(False, raw.get('message', status))\n raise ValueError(f'unrecognized grader status: {status!r}')\n\n\ndef assert_grader_passed(raw: dict) -> None:\n outcome = adapt(raw)\n assert outcome.passed, outcome.detail\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-AST-P1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_invoice_discount.py", + "source": "from invoicing import build_invoice\n\n\ndef test_build_invoice_applies_discount():\n invoice = build_invoice(\n items=[('widget', 2, 1000), ('gadget', 1, 2500)],\n discount_pct=10,\n )\n assert invoice\n assert invoice.total\n assert invoice.lines\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-AST-P1", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G2-AST-P2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "uniqsort_test.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nprintf 'b\\na\\nc\\na\\n' > \"$work/in.txt\"\n./bin/uniqsort \"$work/in.txt\" > \"$work/out.txt\"\n[ -f \"$work/out.txt\" ]\necho \"uniqsort ok\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-AST-P2", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G2-AST-P3", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_cli_version.py", + "source": "import subprocess\nimport sys\n\n\ndef test_cli_reports_version():\n proc = subprocess.run(\n [sys.executable, '-m', 'pkgtool', '--version'],\n capture_output=True,\n text=True,\n )\n assert proc.returncode == 0\n assert '.' in proc.stdout\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-AST-P3", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G2-AST-N1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_cli_errors.py", + "source": "import subprocess\nimport sys\n\n\ndef test_cli_rejects_unknown_flag():\n proc = subprocess.run(\n [sys.executable, '-m', 'pkgtool', '--frobnicate'],\n capture_output=True,\n text=True,\n )\n assert proc.returncode == 2\n assert proc.stdout == ''\n assert proc.stderr.splitlines()[-1] == 'pkgtool: error: unrecognized arguments: --frobnicate'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-AST-N2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_invoice_totals.py", + "source": "from decimal import Decimal\n\nfrom invoicing import build_invoice\n\n\ndef test_build_invoice_applies_discount():\n invoice = build_invoice(items=[('widget', 2, Decimal('10.00'))], discount_pct=10)\n assert invoice.subtotal == Decimal('20.00')\n assert invoice.discount == Decimal('2.00')\n assert invoice.total == Decimal('18.00')\n assert [line.sku for line in invoice.lines] == ['widget']\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-ENV-P1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_report_header.py", + "source": "from datetime import datetime\n\nfrom reports.header import render_header\n\n\ndef test_render_header_includes_generation_time():\n generated = datetime.fromtimestamp(1700000000)\n header = render_header(title='Weekly', generated_at=generated)\n assert header == 'Weekly \u2014 generated 2023-11-14 22:13'\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-ENV-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-ENV-P2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_registry_token.py", + "source": "from pathlib import Path\n\nfrom uploader.auth import token_for\n\n\ndef test_token_for_registry_reads_netrc():\n netrc = Path.home() / '.netrc'\n line = next(\n entry for entry in netrc.read_text().splitlines() if 'registry.internal' in entry\n )\n expected = line.split('password', 1)[1].strip()\n assert token_for('registry.internal') == expected\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-ENV-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-ENV-P3", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_title_sorting.py", + "source": "import locale\n\nfrom catalog.sorting import sort_titles\n\n\ndef test_sort_titles_uses_collation():\n locale.setlocale(locale.LC_COLLATE, '')\n titles = ['zebra', '\u00c4pfel', 'apple', 'Birne']\n assert sort_titles(titles) == ['apple', '\u00c4pfel', 'Birne', 'zebra']\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-ENV-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-ENV-N1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_report_header_utc.py", + "source": "from datetime import datetime, timezone\n\nfrom reports.header import render_header\n\n\ndef test_render_header_includes_generation_time():\n generated = datetime.fromtimestamp(1700000000, tz=timezone.utc)\n header = render_header(title='Weekly', generated_at=generated)\n assert header == 'Weekly \u2014 generated 2023-11-14 22:13 UTC'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-ENV-N2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_registry_token_fixture.py", + "source": "from uploader.auth import token_for\n\n\ndef test_token_for_registry_reads_netrc(tmp_path, monkeypatch):\n monkeypatch.setenv('HOME', str(tmp_path))\n monkeypatch.delenv('NETRC', raising=False)\n netrc = tmp_path / '.netrc'\n netrc.write_text('machine registry.internal login ci password s3cr3t-token\\n')\n netrc.chmod(0o600)\n assert token_for('registry.internal') == 's3cr3t-token'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-GIT-P1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "test_changelog_entries.py", + "source": "import subprocess\n\nfrom changelog import collect_entries\n\n\ndef _git(repo, *args):\n subprocess.run(['git', '-C', str(repo), *args], check=True, capture_output=True)\n\n\ndef test_collect_entries_reads_commit_subjects(tmp_path):\n repo = tmp_path / 'repo'\n repo.mkdir()\n _git(repo, 'init', '-q')\n _git(repo, 'config', 'user.name', 'Fixture')\n _git(repo, 'config', 'user.email', 'fixture@test.local')\n (repo / 'a.txt').write_text('a\\n')\n _git(repo, 'add', 'a.txt')\n _git(repo, 'commit', '-q', '-m', 'feat: add a')\n assert collect_entries(repo) == ['feat: add a']\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-P2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "release_notes_test.sh", + "source": "#!/bin/sh\nset -eu\nrepo=$(mktemp -d)\ntrap 'rm -rf \"$repo\"' EXIT\ngit init -q \"$repo\"\nprintf 'one\\n' > \"$repo/f\"\ngit -C \"$repo\" add f\ngit -C \"$repo\" -c user.name=Fixture -c user.email=fixture@test.local commit -q -m 'feat: one'\nprintf 'two\\n' >> \"$repo/f\"\ngit -C \"$repo\" -c user.name=Fixture -c user.email=fixture@test.local commit -q -am 'fix: two'\nexpected=$(printf '%s\\n' '- fix: two' '- feat: one')\nactual=$(./bin/release-notes \"$repo\")\n[ \"$actual\" = \"$expected\" ] || { printf 'got:\\n%s\\n' \"$actual\" >&2; exit 1; }\necho ok\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-P3", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "test_tag_release.py", + "source": "import subprocess\n\nfrom releaser import tag_release\n\n\ndef test_tag_release_creates_annotated_tag(tmp_path):\n repo = tmp_path / 'r'\n repo.mkdir()\n\n def run(*args):\n return subprocess.run(['git', *args], cwd=repo, check=True, capture_output=True, text=True)\n\n run('init', '-q')\n run('config', 'user.name', 'CI Fixture')\n run('config', 'user.email', 'ci@fixture.test')\n run('config', 'commit.gpgsign', 'false')\n (repo / 'VERSION').write_text('1.4.0\\n')\n run('add', 'VERSION')\n run('commit', '-q', '-m', 'release 1.4.0')\n tag_release(repo, '1.4.0')\n out = run('for-each-ref', '--format=%(objecttype) %(refname:short)', 'refs/tags')\n assert out.stdout.strip() == 'tag v1.4.0'\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-N1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "test_mirror_branch.py", + "source": "import subprocess\n\nfrom sync.mirror import mirror_branch\n\n\ndef _git(cwd, *args):\n return subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef test_mirror_branch_pushes_to_local_remote(tmp_path):\n hooks = tmp_path / 'no-hooks'\n hooks.mkdir()\n origin = tmp_path / 'origin.git'\n _git(tmp_path, 'init', '-q', '--bare', str(origin))\n work = tmp_path / 'work'\n _git(tmp_path, 'init', '-q', '-b', 'main', str(work))\n for key, value in [('user.name', 'Fixture'), ('user.email', 'fixture@test.local'), ('commit.gpgsign', 'false'), ('tag.gpgsign', 'false'), ('core.hooksPath', str(hooks))]:\n _git(work, 'config', key, value)\n _git(work, 'remote', 'add', 'origin', str(origin))\n (work / 'x.txt').write_text('x\\n')\n _git(work, 'add', 'x.txt')\n _git(work, 'commit', '-q', '-m', 'add x')\n mirror_branch(work, 'origin', 'main')\n head = _git(work, 'rev-parse', 'HEAD').strip()\n assert _git(origin, 'rev-parse', 'refs/heads/main').strip() == head\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-GIT-N2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "release_notes_configured.sh", + "source": "#!/bin/sh\nset -eu\nrepo=$(mktemp -d)\ntrap 'rm -rf \"$repo\"' EXIT\ngit init -q \"$repo\"\ngit -C \"$repo\" config user.name 'Fixture'\ngit -C \"$repo\" config user.email 'fixture@test.local'\ngit -C \"$repo\" config commit.gpgsign false\ngit -C \"$repo\" config core.hooksPath /dev/null\nprintf 'one\\n' > \"$repo/f\"\ngit -C \"$repo\" add f\ngit -C \"$repo\" commit -q -m 'feat: one'\nprintf 'two\\n' >> \"$repo/f\"\ngit -C \"$repo\" commit -q -am 'fix: two'\nexpected=$(printf '%s\\n' '- fix: two' '- feat: one')\nactual=$(./bin/release-notes \"$repo\")\n[ \"$actual\" = \"$expected\" ] || { printf 'got:\\n%s\\n' \"$actual\" >&2; exit 1; }\necho ok\n", + "expected_findings": [] + } + ] +} From 0cf900933a44405166632704cdd9a0a1821e603a Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:46:59 +0900 Subject: [PATCH 3/7] fix(review): remove test-integrity env over-reporting and specify command-substitution fail-open Two defects found by scoring a measured run, both reproduced rather than inferred: 1. The previous edit made any host TMPDIR read a Hidden environment dependency, so mktemp/tempfile cases also reported that class on top of Shared temporary directories. The temporary root is now judged only under Shared temporary directories, and inherited pip package-source settings belong to Network isolation when the install can reach an index. 2. A verifier that runs a command inside command substitution and tests empty output reports success when the command is missing or fails. Reproduced: with find absent from PATH, exits 0. The class now names this pattern. Two blind labels were also wrong and are corrected with new case ids: C2-TI-GIT-CONF-N1 -> C2-TI-GIT-CONF-P3 (verifier-fail-open) and C2-TI-G2-GIT-N1 -> C2-TI-G2-GIT-P4 (fixture-git-signing-hooks). Reproduced: with find absent the verifier exits 0; with a global init.templateDir providing post-receive, Initialized empty Git repository in /private/tmp/cl-28/ installs it and the push runs host code. Each corpus records the correction under label_corrections. Constraint: labels change only with executed evidence, never to make a run pass Rejected: relax the 90%/100%/zero-false-positive gate | thresholds belong to the issue Confidence: high Scope-risk: narrow Tested: both mechanisms reproduced in /tmp with isolated HOME; scripts/test-eval-test-integrity-model.sh passes Not-tested: model behaviour under the new wording; validate.sh on this commit; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- skills/reviewing/references/test-integrity.md | 6 ++--- .../review/test-integrity-blind-confirm.json | 22 +++++++++++++--- .../review/test-integrity-blind-gate2.json | 25 +++++++++++++++++-- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/skills/reviewing/references/test-integrity.md b/skills/reviewing/references/test-integrity.md index c051894..c9bb148 100644 --- a/skills/reviewing/references/test-integrity.md +++ b/skills/reviewing/references/test-integrity.md @@ -24,7 +24,7 @@ Evidence includes the path expression and the isolation boundary it bypasses. Do ### Verifier fail-open behavior -Check every verifier and assertion adapter for ignored exit status, swallowed exceptions, default-success fallbacks, unchecked `check=False` results, and missing or unknown verdict handling. A malformed, missing, or infrastructure-error result must fail closed rather than become `clean`, `pass`, or an empty finding list. Severity is P0 when the path authorizes a release or mutation and P1 otherwise. +Check every verifier and assertion adapter for ignored exit status, swallowed exceptions, default-success fallbacks, unchecked `check=False` results, and missing or unknown verdict handling. A command substitution whose failure becomes empty output is one of these: `test -z "$(find ...)"`, `[ -z "$(tool ...)" ]`, or a pipeline whose exit status is not checked reports success when the command itself is missing, fails, or errors. A malformed, missing, or infrastructure-error result must fail closed rather than become `clean`, `pass`, or an empty finding list. Severity is P0 when the path authorizes a release or mutation and P1 otherwise. Evidence traces the failure input to the success path. Do not flag a deliberately handled expected rejection whose result is asserted and whose unknown or infrastructure states remain distinct. @@ -38,7 +38,7 @@ Evidence cites the assertion and the untested distinction. Do not flag a deliber Check whether tests read host `HOME`, `PATH`, `TMPDIR`, locale, timezone, credentials, Git configuration outside fixture repositories, network proxy variables, or tool-specific configuration without setting or clearing them in the fixture. Flag environment reads that change the verdict or target and are not represented in the fixture's declared inputs. Severity is P1 when host state can change authorization or target selection and P2 otherwise. -Evidence names the environment key or implicit process state and the missing fixture declaration. A read of host `HOME`, `PATH`, or `TMPDIR` that the snippet does not first set or clear is a finding even when the snippet shown only stores or prints the value, because the fixture has not declared the input. Do not flag an explicitly supplied value that the fixture records and verifies. +Evidence names the environment key or implicit process state and the missing fixture declaration. A read of host `HOME` or `PATH` that the snippet does not first set or clear is a finding even when the snippet shown only stores or prints the value, because the fixture has not declared the input. The temporary root is an exception: `mktemp`, `tempfile.gettempdir()`, `tempfile.TemporaryDirectory()`, and the process temporary directory are judged only under Shared temporary directories, and the host `TMPDIR`, `TEMP`, or `TMP` they consult is not a separate Hidden environment dependency unless the snippet uses the value for something else. Do not flag an explicitly supplied value that the fixture records and verifies. ### Fixture Git signing/hooks @@ -53,7 +53,7 @@ Report each defect once, under the single class that owns it: - Git identity, signing, hooks, credential helpers, remote settings, and inherited global or system Git configuration of a fixture repository belong to Fixture Git signing/hooks. Do not also report that configuration as a hidden environment dependency. A Git command that actually contacts an external remote, such as `clone`, `fetch`, or `push`, is a separate Network isolation defect. - A configured remote, URL, or host name that the code never contacts is not a network defect. Network isolation needs a call or command that can reach an external host. - An ignored exit status, swallowed exception, or default-success value belongs to Verifier fail-open behavior. Report Assertion specificity only for a test assertion that can pass without the required behavior. -- Hidden environment dependencies covers host state other than Git configuration that the snippet reads or that selects its target or verdict. Running a tool or `git` binary from `PATH` is not by itself an environment defect. +- Hidden environment dependencies covers host state other than Git configuration that the snippet reads or that selects its target or verdict. Running a tool or `git` binary from `PATH` is not by itself an environment defect. Package-source settings such as `PIP_INDEX_URL` or `PIP_CONFIG_FILE` inherited by an install command belong to Network isolation when that command can reach an external index, not to this class. - Judge only the text shown. Do not report a class because unseen helper code might be unsafe. - A snippet is often a fragment written to show one defect. When another class owns that defect, do not also report Assertion specificity because the fragment lacks an assertion; report that class only for an assertion that is present and too weak. diff --git a/tests/conformance/review/test-integrity-blind-confirm.json b/tests/conformance/review/test-integrity-blind-confirm.json index fced708..0b7caed 100644 --- a/tests/conformance/review/test-integrity-blind-confirm.json +++ b/tests/conformance/review/test-integrity-blind-confirm.json @@ -198,13 +198,27 @@ ] }, { - "id": "C2-TI-GIT-CONF-N1", - "class": "fixture-git-signing-hooks", + "id": "C2-TI-GIT-CONF-P3", + "class": "verifier-fail-open", "kind": "holdout", "file": "fixture_test.sh", "source": "#!/bin/sh\nrepo=$(mktemp -d)\nmkdir \"$repo/hooks\"\ngit -C \"$repo\" init\ngit -C \"$repo\" config user.name Tester; git -C \"$repo\" config user.email tester@example.invalid\ngit -C \"$repo\" config commit.gpgsign false; git -C \"$repo\" config core.hooksPath \"$repo/hooks\"\ntest -z \"$(find \"$repo/hooks\" -mindepth 1 -print -quit)\"\ngit -C \"$repo\" config remote.origin.url /dev/null\ngit -C \"$repo\" commit --allow-empty -m fixture", - "expected_findings": [] + "expected_findings": [ + { + "seed_id": "C2-TI-GIT-CONF-P3", + "severity": "P1" + } + ] } ], - "description": "Authored blind to the detectors." + "description": "Authored blind to the detectors.", + "label_corrections": [ + { + "case_id": "C2-TI-GIT-CONF-P3", + "was": "C2-TI-GIT-CONF-N1", + "prior_label": "clean", + "corrected_label": "verifier-fail-open", + "evidence": "reproduced with executed commands, see commit message" + } + ] } diff --git a/tests/conformance/review/test-integrity-blind-gate2.json b/tests/conformance/review/test-integrity-blind-gate2.json index bf1723a..91bafc6 100644 --- a/tests/conformance/review/test-integrity-blind-gate2.json +++ b/tests/conformance/review/test-integrity-blind-gate2.json @@ -317,12 +317,17 @@ ] }, { - "id": "C2-TI-G2-GIT-N1", + "id": "C2-TI-G2-GIT-P4", "class": "fixture-git-signing-hooks", "kind": "holdout", "file": "test_mirror_branch.py", "source": "import subprocess\n\nfrom sync.mirror import mirror_branch\n\n\ndef _git(cwd, *args):\n return subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef test_mirror_branch_pushes_to_local_remote(tmp_path):\n hooks = tmp_path / 'no-hooks'\n hooks.mkdir()\n origin = tmp_path / 'origin.git'\n _git(tmp_path, 'init', '-q', '--bare', str(origin))\n work = tmp_path / 'work'\n _git(tmp_path, 'init', '-q', '-b', 'main', str(work))\n for key, value in [('user.name', 'Fixture'), ('user.email', 'fixture@test.local'), ('commit.gpgsign', 'false'), ('tag.gpgsign', 'false'), ('core.hooksPath', str(hooks))]:\n _git(work, 'config', key, value)\n _git(work, 'remote', 'add', 'origin', str(origin))\n (work / 'x.txt').write_text('x\\n')\n _git(work, 'add', 'x.txt')\n _git(work, 'commit', '-q', '-m', 'add x')\n mirror_branch(work, 'origin', 'main')\n head = _git(work, 'rev-parse', 'HEAD').strip()\n assert _git(origin, 'rev-parse', 'refs/heads/main').strip() == head\n", - "expected_findings": [] + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P4", + "severity": "P1" + } + ] }, { "id": "C2-TI-G2-GIT-N2", @@ -332,5 +337,21 @@ "source": "#!/bin/sh\nset -eu\nrepo=$(mktemp -d)\ntrap 'rm -rf \"$repo\"' EXIT\ngit init -q \"$repo\"\ngit -C \"$repo\" config user.name 'Fixture'\ngit -C \"$repo\" config user.email 'fixture@test.local'\ngit -C \"$repo\" config commit.gpgsign false\ngit -C \"$repo\" config core.hooksPath /dev/null\nprintf 'one\\n' > \"$repo/f\"\ngit -C \"$repo\" add f\ngit -C \"$repo\" commit -q -m 'feat: one'\nprintf 'two\\n' >> \"$repo/f\"\ngit -C \"$repo\" commit -q -am 'fix: two'\nexpected=$(printf '%s\\n' '- fix: two' '- feat: one')\nactual=$(./bin/release-notes \"$repo\")\n[ \"$actual\" = \"$expected\" ] || { printf 'got:\\n%s\\n' \"$actual\" >&2; exit 1; }\necho ok\n", "expected_findings": [] } + ], + "label_corrections": [ + { + "case_id": "C2-TI-GIT-CONF-P3", + "was": "C2-TI-GIT-CONF-N1", + "prior_label": "clean", + "corrected_label": "verifier-fail-open", + "evidence": "reproduced with executed commands, see commit message" + }, + { + "case_id": "C2-TI-G2-GIT-P4", + "was": "C2-TI-G2-GIT-N1", + "prior_label": "clean", + "corrected_label": "fixture-git-signing-hooks", + "evidence": "reproduced with executed commands, see commit message" + } ] } From 72396c3c36ff3d96a1cab16d02b089ed616fc8cb Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:15:13 +0900 Subject: [PATCH 4/7] test(review): add the gate-3 blind corpus for the test-integrity release gate The gate-2 corpus and the four earlier corpora were used to correct the checklist, so they are regression data. This corpus was written by an author who received only the six class descriptions, after the checklist was fixed: 36 cases, 4 defective and 2 clean per class. Two independent judges labeled the sources without seeing the labels; author and judges agree on 35 of 36, and the one dispute (an extra Assertion specificity reading of C2-TI-G3-ENV-P3) is recorded under label_audit instead of being silently resolved. Constraint: out-of-sample evidence needs an author and judges that never saw the checklist or the model output Rejected: reuse gate-2 as out-of-sample | its errors already drove a rule change Confidence: medium Scope-risk: narrow Tested: corpus parses and plans offline; ids unique; 4/2 mix per class verified Not-tested: model behaviour on this corpus; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- .../review/test-integrity-blind-gate3.json | 430 ++++++++++++++++++ 1 file changed, 430 insertions(+) create mode 100644 tests/conformance/review/test-integrity-blind-gate3.json diff --git a/tests/conformance/review/test-integrity-blind-gate3.json b/tests/conformance/review/test-integrity-blind-gate3.json new file mode 100644 index 0000000..4ddadf7 --- /dev/null +++ b/tests/conformance/review/test-integrity-blind-gate3.json @@ -0,0 +1,430 @@ +{ + "schema": "c2-ti/v1", + "description": "Gate-3 blind out-of-sample cases, authored without the checklist or any corpus", + "cases": [ + { + "id": "C2-TI-G3-NET-P1", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/test_fx_convert.py", + "source": "import requests\n\nfrom fxrates.convert import convert_amount\n\n\ndef test_convert_uses_live_rate():\n resp = requests.get(\"https://api.exchangerate.host/latest?base=USD&symbols=EUR\", timeout=10)\n resp.raise_for_status()\n rate = resp.json()[\"rates\"][\"EUR\"]\n result = convert_amount(100, \"USD\", \"EUR\", rate=rate)\n assert result == round(100 * rate, 2)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-P2", + "class": "network-isolation", + "kind": "holdout", + "file": "ci/check_wheel_contents.sh", + "source": "#!/bin/sh\nset -eu\nworkdir=$(mktemp -d)\ntrap 'rm -rf \"$workdir\"' EXIT\npython3 -m venv \"$workdir/venv\"\n\"$workdir/venv/bin/pip\" install --quiet build==1.2.1\n\"$workdir/venv/bin/python\" -m build --wheel --outdir \"$workdir/dist\" .\nwheel=$(ls \"$workdir\"/dist/*.whl)\nunzip -l \"$wheel\" > \"$workdir/listing.txt\"\ngrep -q 'mediakit/__init__.py' \"$workdir/listing.txt\"\ngrep -q 'mediakit/codecs/opus.py' \"$workdir/listing.txt\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-P3", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/test_digest_mailer.py", + "source": "import smtplib\n\nfrom notifier.mailer import build_message\n\n\ndef test_digest_message_is_accepted_by_relay():\n msg = build_message(to=\"ops@example.org\", subject=\"Daily digest\", body=\"3 jobs failed\")\n with smtplib.SMTP(\"smtp.mailgun.org\", 587, timeout=15) as relay:\n relay.ehlo()\n refused = relay.send_message(msg, from_addr=\"noreply@example.org\")\n assert refused == {}\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-P4", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/transcode_opus.sh", + "source": "#!/bin/sh\nset -eu\nscratch=$(mktemp -d)\ntrap 'rm -rf \"$scratch\"' EXIT\ncurl -fsSL -o \"$scratch/sample.flac\" https://samples.example-media.net/audio/sine-440hz.flac\n./bin/transcode --to opus --bitrate 96k \"$scratch/sample.flac\" \"$scratch/out.opus\"\n./bin/probe --field codec \"$scratch/out.opus\" > \"$scratch/codec.txt\"\n./bin/probe --field bitrate \"$scratch/out.opus\" > \"$scratch/bitrate.txt\"\n[ \"$(cat \"$scratch/codec.txt\")\" = \"opus\" ]\n[ \"$(cat \"$scratch/bitrate.txt\")\" = \"96000\" ]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-N1", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/test_fx_client.py", + "source": "from fxrates import client\n\n\nclass FakeResponse:\n def __init__(self, payload):\n self._payload = payload\n self.status_code = 200\n\n def json(self):\n return self._payload\n\n\ndef test_fetch_rate_parses_payload(monkeypatch):\n calls = []\n\n def fake_get(url, timeout):\n calls.append(url)\n return FakeResponse({\"rates\": {\"EUR\": 0.9137}})\n\n monkeypatch.setattr(client.requests, \"get\", fake_get)\n assert client.fetch_rate(\"USD\", \"EUR\") == 0.9137\n assert calls == [\"https://api.exchangerate.host/latest?base=USD&symbols=EUR\"]\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-NET-N2", + "class": "network-isolation", + "kind": "holdout", + "file": "ci/install_from_wheelhouse.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\npython3 -m venv \"$work/venv\"\n\"$work/venv/bin/pip\" --isolated install --no-index --find-links ./tests/wheelhouse --quiet ledgerkit==2.4.0\n\"$work/venv/bin/python\" -c 'import ledgerkit; print(ledgerkit.__version__)' > \"$work/version.txt\"\n[ \"$(cat \"$work/version.txt\")\" = \"2.4.0\" ]\n\"$work/venv/bin/ledgerkit\" balance --input ./tests/data/journal.csv > \"$work/balance.txt\"\ndiff -u ./tests/data/expected_balance.txt \"$work/balance.txt\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-TMP-P1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/test_quarterly_export.py", + "source": "import csv\nimport os\n\nfrom reports.export import export_quarterly\n\n\ndef test_export_writes_header_and_rows():\n out_path = \"/tmp/quarterly_export.csv\"\n export_quarterly(year=2024, quarter=2, destination=out_path)\n with open(out_path, newline=\"\") as fh:\n rows = list(csv.reader(fh))\n os.remove(out_path)\n assert rows[0] == [\"region\", \"revenue\", \"currency\"]\n assert rows[1:] == [[\"EMEA\", \"125000.00\", \"EUR\"], [\"APAC\", \"98000.50\", \"USD\"]]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-P2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/lintcheck_bad_config.sh", + "source": "#!/bin/sh\nset -eu\nout=/tmp/lintcheck-output\nrm -rf \"$out\"\nmkdir -p \"$out\"\n./bin/lintcheck --format json --output \"$out/report.json\" tests/fixtures/bad_config.yaml || status=$?\n[ \"${status:-0}\" -eq 3 ]\npython3 - \"$out/report.json\" <<'EOF'\nimport json, sys\nreport = json.load(open(sys.argv[1]))\nassert [f[\"rule\"] for f in report[\"findings\"]] == [\"duplicate-key\", \"unknown-field\"], report\nEOF\nrm -rf \"$out\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-P3", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/test_dedupe_partitions.py", + "source": "import os\nimport shutil\nimport tempfile\n\nfrom pipeline.stages import dedupe_partitions\n\n\ndef test_dedupe_removes_duplicate_partitions():\n root = os.path.join(tempfile.gettempdir(), \"dedupe-partitions\")\n shutil.rmtree(root, ignore_errors=True)\n os.makedirs(os.path.join(root, \"day=2024-01-01\"))\n os.makedirs(os.path.join(root, \"day=2024-01-01.copy\"))\n os.makedirs(os.path.join(root, \"day=2024-01-02\"))\n removed = dedupe_partitions(root)\n assert removed == [\"day=2024-01-01.copy\"]\n assert sorted(os.listdir(root)) == [\"day=2024-01-01\", \"day=2024-01-02\"]\n shutil.rmtree(root)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-P4", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/thumbnail_resize.sh", + "source": "#!/bin/sh\nset -eu\ndir=$(mktemp -d /tmp/thumbs-XXXXXX)\ncleanup() { rm -rf /tmp/thumbs-*; }\ntrap cleanup EXIT\ncp tests/media/landscape.jpg \"$dir/in.jpg\"\n./bin/thumbnail --width 320 \"$dir/in.jpg\" \"$dir/out.jpg\"\n./bin/imginfo --format '%wx%h' \"$dir/out.jpg\" > \"$dir/size.txt\"\n[ \"$(cat \"$dir/size.txt\")\" = \"320x180\" ]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-N1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/test_layered_config.py", + "source": "import os\nimport tempfile\nimport unittest\n\nfrom configparse.loader import load_layered\n\n\nclass LayeredLoadTest(unittest.TestCase):\n def test_override_file_wins(self):\n with tempfile.TemporaryDirectory(prefix=\"layered-\") as root:\n base = os.path.join(root, \"base.toml\")\n override = os.path.join(root, \"override.toml\")\n with open(base, \"w\") as fh:\n fh.write(\"[server]\\nport = 8080\\nworkers = 2\\n\")\n with open(override, \"w\") as fh:\n fh.write(\"[server]\\nworkers = 8\\n\")\n cfg = load_layered([base, override])\n self.assertEqual(cfg[\"server\"], {\"port\": 8080, \"workers\": 8})\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-TMP-N2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/logsift_level_filter.sh", + "source": "#!/bin/sh\nset -eu\nscratch=$(mktemp -d)\ntrap 'rm -rf \"$scratch\"' EXIT\nmkdir -p \"$scratch/logs\"\nprintf '2024-03-01T10:00:00Z ERROR disk full\\n2024-03-01T10:00:05Z INFO retry\\n' > \"$scratch/logs/app.log\"\n./bin/logsift --level ERROR \"$scratch/logs\" > \"$scratch/out.txt\"\nprintf '2024-03-01T10:00:00Z ERROR disk full\\n' > \"$scratch/expected.txt\"\ncmp \"$scratch/expected.txt\" \"$scratch/out.txt\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-VER-P1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "ci/verify_integration.sh", + "source": "#!/bin/sh\nset -u\nresults_dir=$1\nmkdir -p \"$results_dir\"\npython3 -m pytest -q tests/integration > \"$results_dir/pytest.log\" 2>&1\nfailed=$(grep -Eo '[0-9]+ failed' \"$results_dir/pytest.log\" | cut -d' ' -f1)\nif [ \"${failed:-0}\" -gt 0 ]; then\n echo \"verdict=FAIL\" > \"$results_dir/verdict.txt\"\n exit 1\nfi\necho \"verdict=PASS\" > \"$results_dir/verdict.txt\"\nexit 0\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-P2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "tests/test_invoice_schema.py", + "source": "import json\nimport subprocess\nimport sys\n\n\ndef run_schema_check(payload_path):\n proc = subprocess.run(\n [sys.executable, \"-m\", \"schemacheck\", \"--json\", payload_path],\n capture_output=True,\n text=True,\n )\n try:\n verdict = json.loads(proc.stdout).get(\"verdict\")\n except ValueError:\n verdict = None\n return verdict != \"fail\"\n\n\ndef test_invoice_payload_matches_schema():\n assert run_schema_check(\"tests/fixtures/invoice_v3.json\")\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-P3", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "release/check_signatures.sh", + "source": "#!/bin/sh\nset -eu\nmanifest=dist/release-manifest.json\nlog=$(mktemp)\ntrap 'rm -f \"$log\"' EXIT\n./scripts/verify_signatures.sh \"$manifest\" > \"$log\" 2>&1 || true\nif grep -q 'BAD SIGNATURE' \"$log\"; then\n echo \"signature check: FAILED\"\n exit 1\nfi\necho \"signature check: OK\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-P4", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "tests/test_month_end_ledger.py", + "source": "def check_ledger_balances(ledger_path):\n from finance.ledger import load_ledger\n\n try:\n ledger = load_ledger(ledger_path)\n mismatches = [\n acct for acct in ledger.accounts\n if acct.computed_balance() != acct.reported_balance\n ]\n except Exception:\n mismatches = []\n return {\"status\": \"clean\" if not mismatches else \"dirty\", \"mismatches\": mismatches}\n\n\ndef test_month_end_ledger_is_clean():\n result = check_ledger_balances(\"tests/data/ledger_2024_06.csv\")\n assert result[\"status\"] == \"clean\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-N1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "ci/verify_integration.sh", + "source": "#!/bin/sh\nset -eu\nresults_dir=$1\nmkdir -p \"$results_dir\"\nif python3 -m pytest -q tests/integration > \"$results_dir/pytest.log\" 2>&1; then\n echo \"verdict=PASS\" > \"$results_dir/verdict.txt\"\n exit 0\nfi\necho \"verdict=FAIL\" > \"$results_dir/verdict.txt\"\nexit 1\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-VER-N2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "tests/test_invoice_schema_strict.py", + "source": "import json\nimport subprocess\nimport sys\n\n\ndef run_schema_check(payload_path):\n proc = subprocess.run(\n [sys.executable, \"-m\", \"schemacheck\", \"--json\", payload_path],\n capture_output=True,\n text=True,\n )\n if proc.returncode not in (0, 1):\n raise RuntimeError(f\"schemacheck crashed: {proc.stderr.strip()}\")\n report = json.loads(proc.stdout)\n verdict = report[\"verdict\"]\n if verdict not in (\"pass\", \"fail\"):\n raise RuntimeError(f\"unknown verdict {verdict!r}\")\n return verdict == \"pass\", report[\"errors\"]\n\n\ndef test_invoice_payload_matches_schema():\n ok, errors = run_schema_check(\"tests/fixtures/invoice_v3.json\")\n assert (ok, errors) == (True, [])\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-AST-P1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_media_probe.py", + "source": "from media.probe import probe_file\n\n\ndef test_probe_reports_video_stream_metadata():\n info = probe_file(\"tests/media/clip_1080p_30fps.mp4\")\n video = [s for s in info.streams if s.kind == \"video\"]\n assert video\n assert video[0].width\n assert video[0].height\n assert video[0].frame_rate\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P1", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-P2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/csv2ledger_conversion.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\ncat > \"$work/input.csv\" <<'EOF'\ndate,amount,currency\n2024-01-05,12.50,EUR\n2024-01-06,-3.10,EUR\n2024-01-07,40.00,USD\nEOF\n./bin/csv2ledger --base-currency EUR --rates tests/data/rates.json \"$work/input.csv\" > \"$work/ledger.txt\"\necho \"csv2ledger conversion ok\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P2", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-P3", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_orderctl_validate.py", + "source": "import subprocess\nimport sys\n\n\ndef test_cli_reports_validation_errors():\n proc = subprocess.run(\n [sys.executable, \"-m\", \"orderctl\", \"validate\", \"tests/fixtures/orders_bad.json\"],\n capture_output=True,\n text=True,\n )\n assert proc.returncode == 2\n assert \"error\" in proc.stderr.lower()\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P3", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-P4", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_signup_events.py", + "source": "from webapp.events import EventBus\nfrom webapp.handlers import register_signup_handlers\n\n\ndef test_signup_sends_welcome_email():\n bus = EventBus()\n sent = []\n register_signup_handlers(bus, mailer=sent.append)\n\n def on_sent(event):\n assert event.payload[\"template\"] == \"welcome\"\n assert event.payload[\"to\"] == \"new.user@example.org\"\n\n bus.subscribe(\"email.sent\", on_sent)\n bus.publish(\"user.signed_up\", {\"email\": \"new.user@example.org\"})\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P4", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-N1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_media_probe_exact.py", + "source": "from media.probe import probe_file\n\n\ndef test_probe_reports_video_stream_metadata():\n info = probe_file(\"tests/media/clip_1080p_30fps.mp4\")\n video = [s for s in info.streams if s.kind == \"video\"]\n assert len(video) == 1\n assert (video[0].width, video[0].height) == (1920, 1080)\n assert video[0].frame_rate == 30\n assert video[0].codec == \"h264\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-AST-N2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/orderctl_validate_errors.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nstatus=0\n./bin/orderctl validate tests/fixtures/orders_bad.json > \"$work/out.txt\" 2> \"$work/err.txt\" || status=$?\n[ \"$status\" -eq 2 ]\ncat > \"$work/expected_err.txt\" <<'EOF'\norders_bad.json: order 17: quantity must be positive\norders_bad.json: order 23: unknown sku \"ZX-999\"\nEOF\ndiff -u \"$work/expected_err.txt\" \"$work/err.txt\"\n[ ! -s \"$work/out.txt\" ]\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-ENV-P1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_deploy_target.py", + "source": "import os\n\nfrom deploytool.config import resolve_target\n\n\ndef test_resolve_target_defaults_to_staging():\n cfg_path = os.path.expanduser(\"~/.deploytool/config.toml\")\n target = resolve_target(cli_target=None, config_path=cfg_path)\n assert target.name == \"staging\"\n assert target.region == \"eu-west-1\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-P2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_invoice_due_line.py", + "source": "from datetime import datetime\n\nfrom billing.invoice import render_due_line\n\n\ndef test_due_line_uses_local_calendar_day():\n issued = datetime.fromtimestamp(1717200000)\n line = render_due_line(issued, net_days=30)\n assert line == \"Payment due: 2024-07-01\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-P3", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/wordindex_order.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nprintf 'banana\\nApple\\ncherry\\napple\\n' > \"$work/words.txt\"\n./bin/wordindex \"$work/words.txt\" > \"$work/index.txt\"\nprintf 'Apple\\napple\\nbanana\\ncherry\\n' > \"$work/expected.txt\"\nsort \"$work/words.txt\" | uniq > \"$work/sorted.txt\"\ndiff -u \"$work/sorted.txt\" \"$work/index.txt\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-P4", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_warehouse_connect.py", + "source": "import os\n\n\ndef connection_kwargs(host, dbname, user):\n kwargs = {\"host\": host, \"dbname\": dbname, \"user\": user, \"sslmode\": \"disable\"}\n password = os.environ.get(\"PGPASSWORD\")\n if password:\n kwargs[\"password\"] = password\n return kwargs\n\n\ndef test_connection_kwargs_without_password_uses_peer_auth():\n kwargs = connection_kwargs(host=\"localhost\", dbname=\"analytics\", user=\"etl\")\n assert kwargs == {\"host\": \"localhost\", \"dbname\": \"analytics\", \"user\": \"etl\", \"sslmode\": \"disable\"}\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-N1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_invoice_due_line_utc.py", + "source": "import time\nfrom datetime import datetime\n\nimport pytest\n\nfrom billing.invoice import render_due_line\n\n\n@pytest.fixture\ndef utc_clock(monkeypatch):\n monkeypatch.setenv(\"TZ\", \"UTC\")\n time.tzset()\n yield\n monkeypatch.undo()\n time.tzset()\n\n\ndef test_due_line_uses_local_calendar_day(utc_clock):\n issued = datetime.fromtimestamp(1717200000)\n line = render_due_line(issued, net_days=30)\n assert line == \"Payment due: 2024-07-01\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-ENV-N2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/wordindex_order_c_locale.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nmkdir -p \"$work/home\"\nprintf 'banana\\nApple\\ncherry\\napple\\n' > \"$work/words.txt\"\nLC_ALL=C HOME=\"$work/home\" ./bin/wordindex \"$work/words.txt\" > \"$work/index.txt\"\nprintf 'Apple\\napple\\nbanana\\ncherry\\n' > \"$work/expected.txt\"\ndiff -u \"$work/expected.txt\" \"$work/index.txt\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-GIT-P1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/test_changelog_since_tag.py", + "source": "import subprocess\n\nfrom relnotes.changelog import since_last_tag\n\n\ndef test_changelog_lists_commits_since_tag(tmp_path):\n repo = tmp_path / \"proj\"\n repo.mkdir()\n\n def git(*args):\n subprocess.run([\"git\", *args], cwd=repo, check=True, capture_output=True)\n\n git(\"init\", \"-q\")\n (repo / \"a.txt\").write_text(\"one\\n\")\n git(\"add\", \"a.txt\")\n git(\"commit\", \"-q\", \"-m\", \"feat: first\")\n git(\"tag\", \"v1.0.0\")\n (repo / \"b.txt\").write_text(\"two\\n\")\n git(\"add\", \"b.txt\")\n git(\"commit\", \"-q\", \"-m\", \"fix: second\")\n assert since_last_tag(str(repo)) == [\"fix: second\"]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-P2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/bumpver_minor.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\ngit init -q \"$work/repo\"\ngit -C \"$work/repo\" config user.name \"Fixture Bot\"\ngit -C \"$work/repo\" config user.email \"fixture@example.invalid\"\ngit -C \"$work/repo\" config commit.gpgsign false\nprintf 'version = \"0.3.1\"\\n' > \"$work/repo/pyproject.toml\"\ngit -C \"$work/repo\" add pyproject.toml\ngit -C \"$work/repo\" commit -q -m \"release 0.3.1\"\n./bin/bumpver --repo \"$work/repo\" minor\n[ \"$(git -C \"$work/repo\" log -1 --format=%s)\" = \"release 0.4.0\" ]\ngrep -qx 'version = \"0.4.0\"' \"$work/repo/pyproject.toml\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-P3", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/test_publish_release.py", + "source": "import subprocess\n\nfrom shipit.publish import publish_release\n\n\ndef git(*args, cwd):\n return subprocess.run([\"git\", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef test_publish_pushes_release_branch(tmp_path):\n remote, work = tmp_path / \"origin.git\", tmp_path / \"work\"\n git(\"init\", \"-q\", \"--bare\", str(remote), cwd=tmp_path)\n git(\"init\", \"-q\", str(work), cwd=tmp_path)\n for key, value in [(\"user.name\", \"CI\"), (\"user.email\", \"ci@example.invalid\"),\n (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", \"\")]:\n git(\"config\", key, value, cwd=work)\n (work / \"VERSION\").write_text(\"2.1.0\\n\")\n git(\"add\", \"VERSION\", cwd=work)\n git(\"commit\", \"-q\", \"-m\", \"2.1.0\", cwd=work)\n git(\"remote\", \"add\", \"origin\", str(remote), cwd=work)\n publish_release(str(work), branch=\"release/2.1\")\n assert git(\"rev-parse\", \"release/2.1\", cwd=remote) == git(\"rev-parse\", \"HEAD\", cwd=work)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-P4", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/diffscope_names.sh", + "source": "#!/bin/sh\nset -eu\nfixture=$(mktemp -d)\nout=$(mktemp)\ntrap 'rm -rf \"$fixture\" \"$out\"' EXIT\ngit init -q \"$fixture\"\ngit -C \"$fixture\" config commit.gpgsign false\ngit -C \"$fixture\" config core.hooksPath \"\"\nmkdir -p \"$fixture/src\"\nprintf 'def handler(event):\\n return 200\\n' > \"$fixture/src/app.py\"\ngit -C \"$fixture\" add src/app.py\ngit -C \"$fixture\" commit -q -m \"initial handler\"\nprintf 'def handler(event):\\n return 204\\n' > \"$fixture/src/app.py\"\n./bin/diffscope --repo \"$fixture\" --format names > \"$out\"\n[ \"$(cat \"$out\")\" = \"src/app.py\" ]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-N1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/test_publish_release_isolated.py", + "source": "import subprocess\n\nfrom shipit.publish import publish_release\n\n\ndef git(*args, cwd):\n return subprocess.run([\"git\", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef init_repo(path, bare=False):\n git(\"init\", \"-q\", *([\"--bare\"] if bare else []), str(path), cwd=path.parent)\n for key, value in [(\"user.name\", \"CI\"), (\"user.email\", \"ci@example.invalid\"),\n (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", \"\")]:\n git(\"config\", key, value, cwd=path)\n\n\ndef test_publish_pushes_release_branch(tmp_path):\n remote, work = tmp_path / \"origin.git\", tmp_path / \"work\"\n init_repo(remote, bare=True)\n init_repo(work)\n (work / \"VERSION\").write_text(\"2.1.0\\n\")\n git(\"add\", \"VERSION\", cwd=work)\n git(\"commit\", \"-q\", \"-m\", \"2.1.0\", cwd=work)\n git(\"remote\", \"add\", \"origin\", str(remote), cwd=work)\n publish_release(str(work), branch=\"release/2.1\")\n assert git(\"rev-parse\", \"release/2.1\", cwd=remote) == git(\"rev-parse\", \"HEAD\", cwd=work)\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-GIT-N2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/diffscope_names_isolated.sh", + "source": "#!/bin/sh\nset -eu\nfixture=$(mktemp -d)\nout=$(mktemp)\ntrap 'rm -rf \"$fixture\" \"$out\"' EXIT\ngit init -q \"$fixture\"\ngit -C \"$fixture\" config user.name \"Fixture Bot\"\ngit -C \"$fixture\" config user.email \"fixture@example.invalid\"\ngit -C \"$fixture\" config commit.gpgsign false\ngit -C \"$fixture\" config core.hooksPath \"\"\nmkdir -p \"$fixture/src\"\nprintf 'def handler(event):\\n return 200\\n' > \"$fixture/src/app.py\"\ngit -C \"$fixture\" add src/app.py\ngit -C \"$fixture\" commit -q -m \"initial handler\"\nprintf 'def handler(event):\\n return 204\\n' > \"$fixture/src/app.py\"\n./bin/diffscope --repo \"$fixture\" --format names > \"$out\"\n[ \"$(cat \"$out\")\" = \"src/app.py\" ]\n", + "expected_findings": [] + } + ], + "label_audit": { + "author": "independent author, class descriptions only", + "judges": [ + "reviewer agent", + "scout agent" + ], + "agreement": "35 of 36", + "disputes": [ + { + "case_id": "C2-TI-G3-ENV-P3", + "author": "hidden-environment-dependencies", + "judge_extra": "assertion-specificity", + "resolution": "author label kept; 2 of 3 readings agree" + } + ] + } +} From e313aaadd1aa8b2c9974d93b4c27653b1a31e13f Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:27:59 +0900 Subject: [PATCH 5/7] fix(review): correct the third blind label whose verifier ignores a failed command C2-TI-GIT-FINAL-N1 was labeled clean, but it asserts emptiness with [ -z "$(ls -A dir)" ]. With ls absent from PATH the substitution is empty and the check exits 0, so the fixture proves nothing about the hooks directory. Reproduced with an isolated PATH. Relabeled verifier-fail-open as C2-TI-GIT-FINAL-P3 and recorded under label_corrections. Also verified while checking it: core.hooksPath set to the empty string blocks hooks, and a hooksPath overriding a global init.templateDir also blocks them, so a fixture that sets either form is isolated for that class. Constraint: a label changes only against executed evidence Confidence: high Scope-risk: narrow Tested: ls absent -> verifier exit 0; empty-string and directory hooksPath both block a pre-commit hook installed by init.templateDir Not-tested: model behaviour; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- .../review/test-integrity-blind-final.json | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/conformance/review/test-integrity-blind-final.json b/tests/conformance/review/test-integrity-blind-final.json index e7a3c3f..8e789e1 100644 --- a/tests/conformance/review/test-integrity-blind-final.json +++ b/tests/conformance/review/test-integrity-blind-final.json @@ -199,12 +199,26 @@ ] }, { - "id": "C2-TI-GIT-FINAL-N1", - "class": "fixture-git-signing-hooks", + "id": "C2-TI-GIT-FINAL-P3", + "class": "verifier-fail-open", "kind": "holdout", "file": "fixture_test.sh", "source": "#!/usr/bin/env bash\nset -euo pipefail\nroot=$(mktemp -d)\ntrap 'rm -rf \"$root\"' EXIT\nmkdir \"$root/hooks\"\nexport GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1\ngit init -q \"$root/repo\"\ng() { git -C \"$root/repo\" -c user.name=Fixture -c user.email=fx@example.invalid -c commit.gpgsign=false -c core.hooksPath=\"$root/hooks\" \"$@\"; }\nprintf 'x\\n' > \"$root/repo/f\"\ng add f\ng commit -q -m seed\n[ -z \"$(ls -A \"$root/hooks\")\" ] && [ \"$(g log --format=%an:%s)\" = \"Fixture:seed\" ]\n", - "expected_findings": [] + "expected_findings": [ + { + "seed_id": "C2-TI-GIT-FINAL-P3", + "severity": "P1" + } + ] + } + ], + "label_corrections": [ + { + "case_id": "C2-TI-GIT-FINAL-P3", + "was": "C2-TI-GIT-FINAL-N1", + "prior_label": "clean", + "corrected_label": "verifier-fail-open", + "evidence": "[ -z \"$(ls -A dir)\" ] returns exit 0 when ls is missing, reproduced" } ] -} \ No newline at end of file +} From f453e634edfa1796c65e9348e46692a7cc84ccdd Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:11:24 +0900 Subject: [PATCH 6/7] fix(review): make the test-integrity gate report and corpora match what they claim Independent review of the branch found four defects. 1. The gate documentation listed five corpora and omitted the gate-3 out-of-sample corpus, still called gate-2 out-of-sample, and repeated a 30-of-30 judge agreement that its own label correction made 29 of 30. The doc now lists all six, names gate-3 as out-of-sample, and says separate reporting needs its own plan and score run. 2. The checklist called local Git keys "complete isolation". That is false: a global core.fsmonitor, an attributes-file clean filter, or an init.templateDir can still run host code. Local keys now cover signing and hooks only, and excluding host Git configuration entirely requires GIT_CONFIG_GLOBAL and GIT_CONFIG_NOSYSTEM. 3. Clean cases carried file names that announced the answer ("strict", "utc", "locale", "isolated", "loopback", "configured"), and the file name is part of the request payload. Renamed to neutral ones and recorded under file_name_audit. 4. The headline figures took their counts and per-class detail from repeat 1 while taking precision from the worst repeat. The headline now describes one repeat, identified by headline_repeat, and the dead details list and unreachable unknown-class branch are gone. Label corrections also move to the corpus they belong to, with the reproducing command inline instead of "see commit message". Constraint: measured numbers were taken before the rename, so they are optimistic for gate-3 Confidence: high Scope-risk: narrow Tested: scripts/test-eval-test-integrity-model.sh passes and reverting the headline selection makes it fail; validate.sh passes Not-tested: no re-measurement after the rename, by the user's decision to keep #28 open; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- scripts/eval-test-integrity-model.py | 26 ++++++++----------- scripts/test-eval-test-integrity-model.sh | 4 +++ skills/reviewing/references/test-integrity.md | 7 ++--- .../review/test-integrity-blind-confirm.json | 2 +- .../review/test-integrity-blind-final.json | 2 +- .../review/test-integrity-blind-gate2.json | 22 ++++++---------- .../review/test-integrity-blind-gate3.json | 15 ++++++----- 7 files changed, 37 insertions(+), 41 deletions(-) diff --git a/scripts/eval-test-integrity-model.py b/scripts/eval-test-integrity-model.py index afda1f6..956c470 100755 --- a/scripts/eval-test-integrity-model.py +++ b/scripts/eval-test-integrity-model.py @@ -182,7 +182,6 @@ def score(args): fail("unexpected files in %s: %s" % (response_dir, ", ".join(extra))) predictions = [] malformed = [] - details = [] for response_dir in response_dirs: batch = {} for case in cases: @@ -209,7 +208,6 @@ def evaluate(batch): tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0 per_class = {} details = [] - extra_malformed = [] for class_id in sorted(classes): per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0} for case in cases: @@ -243,9 +241,6 @@ def evaluate(batch): per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1 if not expected and f["severity"] in ("P0", "P1"): p01_negative_fp += 1 - if pred_class not in classes: - # Unknown class identifiers are malformed schema values. - extra_malformed.append({"case_id": case["id"], "reason": "unknown class " + pred_class}) details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)}) precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0) return { @@ -259,19 +254,20 @@ def evaluate(batch): "false_positive_count": fp, "per_class": per_class, "cases": details, - "malformed_extra": extra_malformed, } repeats = [evaluate(batch) for batch in predictions] - for item in repeats: - malformed.extend(item.pop("malformed_extra")) - # The gate must hold in every repeat; the headline figures are the worst repeat. - worst = dict(repeats[0]) - worst["positive_recall"] = min(item["positive_recall"] for item in repeats) - worst["p0_p1_recall"] = min(item["p0_p1_recall"] for item in repeats) - worst["precision"] = min(item["precision"] for item in repeats) - worst["clean_negative_p0_p1_false_positives"] = max(item["clean_negative_p0_p1_false_positives"] for item in repeats) - report = dict(worst, malformed_responses=malformed, repeat_mismatches=repeat_mismatch) + # The gate must hold in every repeat. The headline is the repeat with the + # worst precision, so its per-class figures, case list and counts agree with + # the headline precision instead of describing another repeat. + headline = min(range(len(repeats)), key=lambda index: (repeats[index]["precision"], -repeats[index]["clean_negative_p0_p1_false_positives"], -repeats[index]["false_positive_count"])) + report = dict(repeats[headline]) + report["positive_recall"] = min(item["positive_recall"] for item in repeats) + report["p0_p1_recall"] = min(item["p0_p1_recall"] for item in repeats) + report["precision"] = min(item["precision"] for item in repeats) + report["clean_negative_p0_p1_false_positives"] = max(item["clean_negative_p0_p1_false_positives"] for item in repeats) + report["headline_repeat"] = headline + 1 + report = dict(report, malformed_responses=malformed, repeat_mismatches=repeat_mismatch) report["repeats"] = [{key: item[key] for key in ("positive_recall", "p0_p1_recall", "precision", "clean_negative_p0_p1_false_positives", "false_positive_count", "true_positive_count")} for item in repeats] print(json.dumps(report, indent=2, sort_keys=True)) if report["p0_p1_recall"] < 1.0 or report["precision"] < 0.90 or report["clean_negative_p0_p1_false_positives"] or malformed or repeat_mismatch: diff --git a/scripts/test-eval-test-integrity-model.sh b/scripts/test-eval-test-integrity-model.sh index 8164fa8..207dfb7 100755 --- a/scripts/test-eval-test-integrity-model.sh +++ b/scripts/test-eval-test-integrity-model.sh @@ -172,12 +172,16 @@ with tempfile.TemporaryDirectory(prefix="c2ti-model-test-") as temporary: extra_report = json.loads(extra_class.stdout) assert extra_report["repeat_mismatches"] == [], extra_class.stdout assert [item["precision"] for item in extra_report["repeats"]] == [1.0, 0.5], extra_class.stdout + # The headline describes the same repeat as the headline precision. + assert extra_report["headline_repeat"] == 2, extra_class.stdout + assert extra_report["false_positive_count"] == 1 and extra_report["true_positive_count"] == 1, extra_class.stdout # A clean-case P0/P1 false positive in a later repeat fails the gate. late_fp = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("late-a", [hit], [])), str(write_repeat("late-b", [hit], [clean_fp])), "--corpus", str(corpus), "--repeat", "2"], expected=1) late_report = json.loads(late_fp.stdout) assert late_report["repeat_mismatches"] == [], late_fp.stdout assert [item["clean_negative_p0_p1_false_positives"] for item in late_report["repeats"]] == [0, 1], late_fp.stdout assert late_report["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout + assert late_report["headline_repeat"] == 2, late_fp.stdout # Identical passing repeats pass. call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("same-a", [hit], [])), str(write_repeat("same-b", [hit], [])), "--corpus", str(corpus), "--repeat", "2"]) # The same directory cannot stand in for several repeats. diff --git a/skills/reviewing/references/test-integrity.md b/skills/reviewing/references/test-integrity.md index c9bb148..b9dffbc 100644 --- a/skills/reviewing/references/test-integrity.md +++ b/skills/reviewing/references/test-integrity.md @@ -44,7 +44,7 @@ Evidence names the environment key or implicit process state and the missing fix Check every temporary Git repository for explicit identity, signing policy, hooks path, and remote configuration. A fixture must not inherit host signing keys, global hooks, credential helpers, or remotes. Flag missing `commit.gpgsign=false` when the fixture must commit without host keys, missing empty `core.hooksPath`, and use of an inherited global configuration. Severity is P1 when host code can execute or a commit cannot be reproduced; otherwise P2. -Evidence cites the Git setup and the inherited state that remains possible. Do not flag a fixture that sets identity, `commit.gpgsign=false`, and `core.hooksPath` explicitly with `--local` and proves its hooks directory is empty; setting those keys locally is complete isolation for this class, and a missing `GIT_CONFIG_GLOBAL` or `GIT_CONFIG_NOSYSTEM` is not a separate finding then. +Evidence cites the Git setup and the inherited state that remains possible. Do not flag a fixture that sets identity and `commit.gpgsign=false` locally and takes one of these hook-isolation forms: an empty `core.hooksPath`, a `core.hooksPath` directory the fixture created and does not populate, or `core.hooksPath=/dev/null`. Those keys isolate signing and hooks only. Other inherited Git configuration can still execute host code, so do not describe local keys as complete isolation: a global `core.fsmonitor`, `core.attributesFile` with a `filter..clean` command, or a global `init.templateDir` remains reachable. Flag that residual only when host code can execute through it, and treat a fixture that must exclude host Git configuration entirely as requiring `GIT_CONFIG_GLOBAL` and `GIT_CONFIG_NOSYSTEM`. ### Class boundaries @@ -77,7 +77,7 @@ Run `bash scripts/test-review-test-integrity.sh seeded` to exercise seed and hol ## Model release gate -The release gate scores the real model reviewer against five corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, `tests/conformance/review/test-integrity-blind-final.json`, and `tests/conformance/review/test-integrity-blind-gate2.json`. The blind corpora were authored without access to the detectors. The first four were used to tune the checklist wording, so after that tuning they are regression data, not out-of-sample evidence. `test-integrity-blind-gate2.json` was written by an author who had only the class descriptions, before the latest checklist edit that responded to the measured errors; two independent judges, labeling from the sources alone, agreed with all 30 of its labels. Report its results separately as the out-of-sample measurement. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. +The release gate scores the real model reviewer against six corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, `tests/conformance/review/test-integrity-blind-final.json`, `tests/conformance/review/test-integrity-blind-gate2.json`, and `tests/conformance/review/test-integrity-blind-gate3.json`. The blind corpora were authored without access to the detectors. The first five were used to tune or correct the checklist, so they are regression data, not out-of-sample evidence. `test-integrity-blind-gate3.json` is the out-of-sample corpus: it was written after the checklist was fixed, by an author who had only the class descriptions. Two independent judges labeled its sources without seeing the labels and agreed with 35 of its 36; the one dispute is recorded under `label_audit`. Report its figures separately, which needs a separate plan and score run containing only that corpus, because `score` has no per-corpus breakdown and class severities are declared only in `test-integrity-corpus.json`. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. First create the exact request plan without invoking a model: @@ -88,6 +88,7 @@ python3 scripts/eval-test-integrity-model.py plan \ --corpus tests/conformance/review/test-integrity-blind-confirm.json \ --corpus tests/conformance/review/test-integrity-blind-final.json \ --corpus tests/conformance/review/test-integrity-blind-gate2.json \ + --corpus tests/conformance/review/test-integrity-blind-gate3.json \ --out /tmp/c2ti-model-plan shasum -a 256 /tmp/c2ti-model-plan/plan.json ``` @@ -105,4 +106,4 @@ python3 scripts/eval-test-integrity-model.py run \ --approval ``` -For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all five `--corpus` arguments to `score` as in `plan`. The gate requires, in every repeat, 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, and no malformed or missing responses. Determinism requires the same set of detected seed ids in every repeat; free-text wording and extra findings do not count toward it, but extra findings still lower each repeat's precision. The report shows the worst repeat as its headline figures, lists each repeat under `repeats`, and also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. +For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all six `--corpus` arguments to `score` as in `plan`. The gate requires, in every repeat, 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, and no malformed or missing responses. Determinism requires the same set of detected seed ids in every repeat; free-text wording and extra findings do not count toward it, but extra findings still lower each repeat's precision. The report shows the worst repeat as its headline figures, lists each repeat under `repeats`, and also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. diff --git a/tests/conformance/review/test-integrity-blind-confirm.json b/tests/conformance/review/test-integrity-blind-confirm.json index 0b7caed..5bf91a9 100644 --- a/tests/conformance/review/test-integrity-blind-confirm.json +++ b/tests/conformance/review/test-integrity-blind-confirm.json @@ -218,7 +218,7 @@ "was": "C2-TI-GIT-CONF-N1", "prior_label": "clean", "corrected_label": "verifier-fail-open", - "evidence": "reproduced with executed commands, see commit message" + "evidence": "/bin/sh without set -e; test -z \"$(find dir -mindepth 1 -print -quit)\" exits 0 when find is missing from PATH, and the script ignores that result" } ] } diff --git a/tests/conformance/review/test-integrity-blind-final.json b/tests/conformance/review/test-integrity-blind-final.json index 8e789e1..1d548da 100644 --- a/tests/conformance/review/test-integrity-blind-final.json +++ b/tests/conformance/review/test-integrity-blind-final.json @@ -218,7 +218,7 @@ "was": "C2-TI-GIT-FINAL-N1", "prior_label": "clean", "corrected_label": "verifier-fail-open", - "evidence": "[ -z \"$(ls -A dir)\" ] returns exit 0 when ls is missing, reproduced" + "evidence": "[ -z \"$(ls -A dir)\" ] exits 0 when ls is missing from PATH, so the check reports an empty hooks directory without inspecting it" } ] } diff --git a/tests/conformance/review/test-integrity-blind-gate2.json b/tests/conformance/review/test-integrity-blind-gate2.json index 91bafc6..f9e759a 100644 --- a/tests/conformance/review/test-integrity-blind-gate2.json +++ b/tests/conformance/review/test-integrity-blind-gate2.json @@ -53,7 +53,7 @@ "id": "C2-TI-G2-NET-N2", "class": "network-isolation", "kind": "holdout", - "file": "test_tcp_banner_loopback.py", + "file": "test_tcp_banner_port.py", "source": "import socket\nimport threading\n\nfrom netprobe import tcp_banner\n\n\ndef test_tcp_banner_reads_first_line_from_loopback():\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv:\n srv.bind(('127.0.0.1', 0))\n srv.listen(1)\n port = srv.getsockname()[1]\n\n def serve():\n conn, _ = srv.accept()\n with conn:\n conn.sendall(b'220 mail.test ESMTP ready\\r\\n')\n\n worker = threading.Thread(target=serve)\n worker.start()\n banner = tcp_banner('127.0.0.1', port, timeout=5.0)\n worker.join(timeout=5)\n assert banner == '220 mail.test ESMTP ready'\n", "expected_findings": [] }, @@ -155,7 +155,7 @@ "id": "C2-TI-G2-VER-N1", "class": "verifier-fail-open", "kind": "holdout", - "file": "verify_schema_strict.py", + "file": "verify_schema_report.py", "source": "import json\nimport subprocess\nimport sys\n\ndef verify(artifact_dir: str) -> dict:\n try:\n proc = subprocess.run(\n [sys.executable, '-m', 'schema_check', artifact_dir],\n capture_output=True, text=True, timeout=120,\n )\n except subprocess.TimeoutExpired:\n return {'verdict': 'error', 'reason': 'timeout'}\n if proc.returncode not in (0, 1):\n return {'verdict': 'error', 'reason': f'exit {proc.returncode}'}\n try:\n violations = json.loads(proc.stdout)['violations']\n except (json.JSONDecodeError, KeyError, TypeError):\n return {'verdict': 'error', 'reason': 'unparseable report'}\n verdict = 'pass' if violations == [] and proc.returncode == 0 else 'fail'\n return {'verdict': verdict, 'violations': violations}\n\nif __name__ == '__main__':\n result = verify(sys.argv[1])\n print(json.dumps(result))\n sys.exit(0 if result['verdict'] == 'pass' else 1)\n", "expected_findings": [] }, @@ -163,7 +163,7 @@ "id": "C2-TI-G2-VER-N2", "class": "verifier-fail-open", "kind": "holdout", - "file": "grader_outcome.py", + "file": "grader_outcome_adapter.py", "source": "from dataclasses import dataclass\n\n\n@dataclass\nclass Outcome:\n passed: bool\n detail: str\n\n\ndef adapt(raw: dict) -> Outcome:\n status = raw.get('status')\n if status == 'pass':\n return Outcome(True, raw.get('message', 'pass'))\n if status in ('fail', 'error'):\n return Outcome(False, raw.get('message', status))\n raise ValueError(f'unrecognized grader status: {status!r}')\n\n\ndef assert_grader_passed(raw: dict) -> None:\n outcome = adapt(raw)\n assert outcome.passed, outcome.detail\n", "expected_findings": [] }, @@ -265,7 +265,7 @@ "id": "C2-TI-G2-ENV-N1", "class": "hidden-environment-dependencies", "kind": "holdout", - "file": "test_report_header_utc.py", + "file": "test_report_header_rendering.py", "source": "from datetime import datetime, timezone\n\nfrom reports.header import render_header\n\n\ndef test_render_header_includes_generation_time():\n generated = datetime.fromtimestamp(1700000000, tz=timezone.utc)\n header = render_header(title='Weekly', generated_at=generated)\n assert header == 'Weekly \u2014 generated 2023-11-14 22:13 UTC'\n", "expected_findings": [] }, @@ -333,25 +333,19 @@ "id": "C2-TI-G2-GIT-N2", "class": "fixture-git-signing-hooks", "kind": "holdout", - "file": "release_notes_configured.sh", + "file": "release_notes_existing.sh", "source": "#!/bin/sh\nset -eu\nrepo=$(mktemp -d)\ntrap 'rm -rf \"$repo\"' EXIT\ngit init -q \"$repo\"\ngit -C \"$repo\" config user.name 'Fixture'\ngit -C \"$repo\" config user.email 'fixture@test.local'\ngit -C \"$repo\" config commit.gpgsign false\ngit -C \"$repo\" config core.hooksPath /dev/null\nprintf 'one\\n' > \"$repo/f\"\ngit -C \"$repo\" add f\ngit -C \"$repo\" commit -q -m 'feat: one'\nprintf 'two\\n' >> \"$repo/f\"\ngit -C \"$repo\" commit -q -am 'fix: two'\nexpected=$(printf '%s\\n' '- fix: two' '- feat: one')\nactual=$(./bin/release-notes \"$repo\")\n[ \"$actual\" = \"$expected\" ] || { printf 'got:\\n%s\\n' \"$actual\" >&2; exit 1; }\necho ok\n", "expected_findings": [] } ], "label_corrections": [ - { - "case_id": "C2-TI-GIT-CONF-P3", - "was": "C2-TI-GIT-CONF-N1", - "prior_label": "clean", - "corrected_label": "verifier-fail-open", - "evidence": "reproduced with executed commands, see commit message" - }, { "case_id": "C2-TI-G2-GIT-P4", "was": "C2-TI-G2-GIT-N1", "prior_label": "clean", "corrected_label": "fixture-git-signing-hooks", - "evidence": "reproduced with executed commands, see commit message" + "evidence": "global init.templateDir with an executable hooks/post-receive, then git init --bare origin, then push: the hook file is installed in origin.git/hooks and runs on push" } - ] + ], + "file_name_audit": "Case file names avoid safety-bearing words (strict, exact, utc, locale, isolated, loopback, configured) so the payload never hints at the expected verdict." } diff --git a/tests/conformance/review/test-integrity-blind-gate3.json b/tests/conformance/review/test-integrity-blind-gate3.json index 4ddadf7..bfa0d6d 100644 --- a/tests/conformance/review/test-integrity-blind-gate3.json +++ b/tests/conformance/review/test-integrity-blind-gate3.json @@ -202,7 +202,7 @@ "id": "C2-TI-G3-VER-N2", "class": "verifier-fail-open", "kind": "holdout", - "file": "tests/test_invoice_schema_strict.py", + "file": "tests/test_invoice_schema_check.py", "source": "import json\nimport subprocess\nimport sys\n\n\ndef run_schema_check(payload_path):\n proc = subprocess.run(\n [sys.executable, \"-m\", \"schemacheck\", \"--json\", payload_path],\n capture_output=True,\n text=True,\n )\n if proc.returncode not in (0, 1):\n raise RuntimeError(f\"schemacheck crashed: {proc.stderr.strip()}\")\n report = json.loads(proc.stdout)\n verdict = report[\"verdict\"]\n if verdict not in (\"pass\", \"fail\"):\n raise RuntimeError(f\"unknown verdict {verdict!r}\")\n return verdict == \"pass\", report[\"errors\"]\n\n\ndef test_invoice_payload_matches_schema():\n ok, errors = run_schema_check(\"tests/fixtures/invoice_v3.json\")\n assert (ok, errors) == (True, [])\n", "expected_findings": [] }, @@ -262,7 +262,7 @@ "id": "C2-TI-G3-AST-N1", "class": "assertion-specificity", "kind": "holdout", - "file": "tests/test_media_probe_exact.py", + "file": "tests/test_media_probe_fields.py", "source": "from media.probe import probe_file\n\n\ndef test_probe_reports_video_stream_metadata():\n info = probe_file(\"tests/media/clip_1080p_30fps.mp4\")\n video = [s for s in info.streams if s.kind == \"video\"]\n assert len(video) == 1\n assert (video[0].width, video[0].height) == (1920, 1080)\n assert video[0].frame_rate == 30\n assert video[0].codec == \"h264\"\n", "expected_findings": [] }, @@ -330,7 +330,7 @@ "id": "C2-TI-G3-ENV-N1", "class": "hidden-environment-dependencies", "kind": "holdout", - "file": "tests/test_invoice_due_line_utc.py", + "file": "tests/test_invoice_due_line.py", "source": "import time\nfrom datetime import datetime\n\nimport pytest\n\nfrom billing.invoice import render_due_line\n\n\n@pytest.fixture\ndef utc_clock(monkeypatch):\n monkeypatch.setenv(\"TZ\", \"UTC\")\n time.tzset()\n yield\n monkeypatch.undo()\n time.tzset()\n\n\ndef test_due_line_uses_local_calendar_day(utc_clock):\n issued = datetime.fromtimestamp(1717200000)\n line = render_due_line(issued, net_days=30)\n assert line == \"Payment due: 2024-07-01\"\n", "expected_findings": [] }, @@ -338,7 +338,7 @@ "id": "C2-TI-G3-ENV-N2", "class": "hidden-environment-dependencies", "kind": "holdout", - "file": "tests/wordindex_order_c_locale.sh", + "file": "tests/wordindex_order.sh", "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nmkdir -p \"$work/home\"\nprintf 'banana\\nApple\\ncherry\\napple\\n' > \"$work/words.txt\"\nLC_ALL=C HOME=\"$work/home\" ./bin/wordindex \"$work/words.txt\" > \"$work/index.txt\"\nprintf 'Apple\\napple\\nbanana\\ncherry\\n' > \"$work/expected.txt\"\ndiff -u \"$work/expected.txt\" \"$work/index.txt\"\n", "expected_findings": [] }, @@ -398,7 +398,7 @@ "id": "C2-TI-G3-GIT-N1", "class": "fixture-git-signing-hooks", "kind": "holdout", - "file": "tests/test_publish_release_isolated.py", + "file": "tests/test_publish_release.py", "source": "import subprocess\n\nfrom shipit.publish import publish_release\n\n\ndef git(*args, cwd):\n return subprocess.run([\"git\", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef init_repo(path, bare=False):\n git(\"init\", \"-q\", *([\"--bare\"] if bare else []), str(path), cwd=path.parent)\n for key, value in [(\"user.name\", \"CI\"), (\"user.email\", \"ci@example.invalid\"),\n (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", \"\")]:\n git(\"config\", key, value, cwd=path)\n\n\ndef test_publish_pushes_release_branch(tmp_path):\n remote, work = tmp_path / \"origin.git\", tmp_path / \"work\"\n init_repo(remote, bare=True)\n init_repo(work)\n (work / \"VERSION\").write_text(\"2.1.0\\n\")\n git(\"add\", \"VERSION\", cwd=work)\n git(\"commit\", \"-q\", \"-m\", \"2.1.0\", cwd=work)\n git(\"remote\", \"add\", \"origin\", str(remote), cwd=work)\n publish_release(str(work), branch=\"release/2.1\")\n assert git(\"rev-parse\", \"release/2.1\", cwd=remote) == git(\"rev-parse\", \"HEAD\", cwd=work)\n", "expected_findings": [] }, @@ -406,7 +406,7 @@ "id": "C2-TI-G3-GIT-N2", "class": "fixture-git-signing-hooks", "kind": "holdout", - "file": "tests/diffscope_names_isolated.sh", + "file": "tests/diffscope_names.sh", "source": "#!/bin/sh\nset -eu\nfixture=$(mktemp -d)\nout=$(mktemp)\ntrap 'rm -rf \"$fixture\" \"$out\"' EXIT\ngit init -q \"$fixture\"\ngit -C \"$fixture\" config user.name \"Fixture Bot\"\ngit -C \"$fixture\" config user.email \"fixture@example.invalid\"\ngit -C \"$fixture\" config commit.gpgsign false\ngit -C \"$fixture\" config core.hooksPath \"\"\nmkdir -p \"$fixture/src\"\nprintf 'def handler(event):\\n return 200\\n' > \"$fixture/src/app.py\"\ngit -C \"$fixture\" add src/app.py\ngit -C \"$fixture\" commit -q -m \"initial handler\"\nprintf 'def handler(event):\\n return 204\\n' > \"$fixture/src/app.py\"\n./bin/diffscope --repo \"$fixture\" --format names > \"$out\"\n[ \"$(cat \"$out\")\" = \"src/app.py\" ]\n", "expected_findings": [] } @@ -426,5 +426,6 @@ "resolution": "author label kept; 2 of 3 readings agree" } ] - } + }, + "file_name_audit": "Case file names avoid safety-bearing words (strict, exact, utc, locale, isolated, loopback, configured) so the payload never hints at the expected verdict." } From 1574d9ec39ba7a1898d2abb3a6a54ee74f730fdf Mon Sep 17 00:00:00 2001 From: Jaehoon You <158752+teslamint@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:33:12 +0900 Subject: [PATCH 7/7] fix(review): score every test-integrity corpus alone and keep worst figures separate Three findings from the CodeRabbit review of this branch. 1. Class severities were declared only in test-integrity-corpus.json, so scoring any single blind corpus alone used an empty class list: every non-empty response counted as malformed, recall fell to zero, and P0/P1 recall then read 100% because it had no expected positives. Every corpus now declares the six classes, and scoring test-integrity-blind-gate3.json alone reports recall and precision instead of a vacuous pass. 2. The headline copied the worst-precision repeat but overwrote its recall with the minimum across repeats, so the headline recall could disagree with its own true-positive count and case list. The headline now keeps its repeat's own figures, names the repeat, and exposes the minima and maximum under worst, which the gate uses. 3. The new boundary told reviewers not to report a missing assertion, which suppressed a real defect in a complete test that never verifies what it names. Missing verification in a complete test is reportable again; only fragments that exist to show another class are exempt. Confidence: high Scope-risk: narrow Tested: scripts/test-eval-test-integrity-model.sh passes and asserts headline-versus-repeat consistency; scoring gate-3 alone now reports precision 0.80 and worst recall 0.917 instead of all-malformed; validate.sh passes Not-tested: no model re-measurement, by the user's decision to keep #28 open; no CHANGELOG entry because compound-loop writes its CHANGELOG at release time and has no Unreleased section Assisted-by: Claude Sonnet 5.5 with omp --- scripts/eval-test-integrity-model.py | 15 ++++++---- scripts/test-eval-test-integrity-model.sh | 6 +++- skills/reviewing/references/test-integrity.md | 4 +-- .../review/test-integrity-blind-confirm.json | 28 ++++++++++++++++- .../review/test-integrity-blind-dev.json | 30 +++++++++++++++++-- .../review/test-integrity-blind-final.json | 26 ++++++++++++++++ .../review/test-integrity-blind-gate2.json | 26 ++++++++++++++++ .../review/test-integrity-blind-gate3.json | 26 ++++++++++++++++ 8 files changed, 150 insertions(+), 11 deletions(-) diff --git a/scripts/eval-test-integrity-model.py b/scripts/eval-test-integrity-model.py index 956c470..79f594e 100755 --- a/scripts/eval-test-integrity-model.py +++ b/scripts/eval-test-integrity-model.py @@ -262,15 +262,20 @@ def evaluate(batch): # the headline precision instead of describing another repeat. headline = min(range(len(repeats)), key=lambda index: (repeats[index]["precision"], -repeats[index]["clean_negative_p0_p1_false_positives"], -repeats[index]["false_positive_count"])) report = dict(repeats[headline]) - report["positive_recall"] = min(item["positive_recall"] for item in repeats) - report["p0_p1_recall"] = min(item["p0_p1_recall"] for item in repeats) - report["precision"] = min(item["precision"] for item in repeats) - report["clean_negative_p0_p1_false_positives"] = max(item["clean_negative_p0_p1_false_positives"] for item in repeats) report["headline_repeat"] = headline + 1 + # Worst figures are reported separately so every headline count keeps + # describing the headline repeat. + report["worst"] = { + "positive_recall": min(item["positive_recall"] for item in repeats), + "p0_p1_recall": min(item["p0_p1_recall"] for item in repeats), + "precision": min(item["precision"] for item in repeats), + "clean_negative_p0_p1_false_positives": max(item["clean_negative_p0_p1_false_positives"] for item in repeats), + } report = dict(report, malformed_responses=malformed, repeat_mismatches=repeat_mismatch) report["repeats"] = [{key: item[key] for key in ("positive_recall", "p0_p1_recall", "precision", "clean_negative_p0_p1_false_positives", "false_positive_count", "true_positive_count")} for item in repeats] print(json.dumps(report, indent=2, sort_keys=True)) - if report["p0_p1_recall"] < 1.0 or report["precision"] < 0.90 or report["clean_negative_p0_p1_false_positives"] or malformed or repeat_mismatch: + worst = report["worst"] + if worst["p0_p1_recall"] < 1.0 or worst["precision"] < 0.90 or worst["clean_negative_p0_p1_false_positives"] or malformed or repeat_mismatch: raise SystemExit(1) diff --git a/scripts/test-eval-test-integrity-model.sh b/scripts/test-eval-test-integrity-model.sh index 207dfb7..3b52b35 100755 --- a/scripts/test-eval-test-integrity-model.sh +++ b/scripts/test-eval-test-integrity-model.sh @@ -175,13 +175,17 @@ with tempfile.TemporaryDirectory(prefix="c2ti-model-test-") as temporary: # The headline describes the same repeat as the headline precision. assert extra_report["headline_repeat"] == 2, extra_class.stdout assert extra_report["false_positive_count"] == 1 and extra_report["true_positive_count"] == 1, extra_class.stdout + # Every headline figure describes the headline repeat; worst figures are separate. + assert extra_report["positive_recall"] == extra_report["repeats"][1]["positive_recall"], extra_class.stdout + assert extra_report["worst"]["precision"] == 0.5, extra_class.stdout # A clean-case P0/P1 false positive in a later repeat fails the gate. late_fp = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("late-a", [hit], [])), str(write_repeat("late-b", [hit], [clean_fp])), "--corpus", str(corpus), "--repeat", "2"], expected=1) late_report = json.loads(late_fp.stdout) assert late_report["repeat_mismatches"] == [], late_fp.stdout assert [item["clean_negative_p0_p1_false_positives"] for item in late_report["repeats"]] == [0, 1], late_fp.stdout - assert late_report["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout + assert late_report["worst"]["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout assert late_report["headline_repeat"] == 2, late_fp.stdout + assert late_report["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout # Identical passing repeats pass. call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("same-a", [hit], [])), str(write_repeat("same-b", [hit], [])), "--corpus", str(corpus), "--repeat", "2"]) # The same directory cannot stand in for several repeats. diff --git a/skills/reviewing/references/test-integrity.md b/skills/reviewing/references/test-integrity.md index b9dffbc..4d2b430 100644 --- a/skills/reviewing/references/test-integrity.md +++ b/skills/reviewing/references/test-integrity.md @@ -55,7 +55,7 @@ Report each defect once, under the single class that owns it: - An ignored exit status, swallowed exception, or default-success value belongs to Verifier fail-open behavior. Report Assertion specificity only for a test assertion that can pass without the required behavior. - Hidden environment dependencies covers host state other than Git configuration that the snippet reads or that selects its target or verdict. Running a tool or `git` binary from `PATH` is not by itself an environment defect. Package-source settings such as `PIP_INDEX_URL` or `PIP_CONFIG_FILE` inherited by an install command belong to Network isolation when that command can reach an external index, not to this class. - Judge only the text shown. Do not report a class because unseen helper code might be unsafe. -- A snippet is often a fragment written to show one defect. When another class owns that defect, do not also report Assertion specificity because the fragment lacks an assertion; report that class only for an assertion that is present and too weak. +- Report a missing check in a complete test when the contract needs one: a test whose body never asserts or verifies the behavior it names is an Assertion specificity finding, and so is a test that inspects a result it never checks. A snippet is often a fragment written to show a different defect; when another class owns that defect, do not also report Assertion specificity merely because the fragment has no assertion. ## Per-run evidence record @@ -77,7 +77,7 @@ Run `bash scripts/test-review-test-integrity.sh seeded` to exercise seed and hol ## Model release gate -The release gate scores the real model reviewer against six corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, `tests/conformance/review/test-integrity-blind-final.json`, `tests/conformance/review/test-integrity-blind-gate2.json`, and `tests/conformance/review/test-integrity-blind-gate3.json`. The blind corpora were authored without access to the detectors. The first five were used to tune or correct the checklist, so they are regression data, not out-of-sample evidence. `test-integrity-blind-gate3.json` is the out-of-sample corpus: it was written after the checklist was fixed, by an author who had only the class descriptions. Two independent judges labeled its sources without seeing the labels and agreed with 35 of its 36; the one dispute is recorded under `label_audit`. Report its figures separately, which needs a separate plan and score run containing only that corpus, because `score` has no per-corpus breakdown and class severities are declared only in `test-integrity-corpus.json`. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. +The release gate scores the real model reviewer against six corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, `tests/conformance/review/test-integrity-blind-final.json`, `tests/conformance/review/test-integrity-blind-gate2.json`, and `tests/conformance/review/test-integrity-blind-gate3.json`. The blind corpora were authored without access to the detectors. The first five were used to tune or correct the checklist, so they are regression data, not out-of-sample evidence. `test-integrity-blind-gate3.json` is the out-of-sample corpus: it was written after the checklist was fixed, by an author who had only the class descriptions. Two independent judges labeled its sources without seeing the labels and agreed with 35 of its 36; the one dispute is recorded under `label_audit`. Report its figures separately with a plan and score run containing only that corpus; every corpus declares the six class severities, so a single-corpus run scores correctly. `score` itself has no per-corpus breakdown. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. First create the exact request plan without invoking a model: diff --git a/tests/conformance/review/test-integrity-blind-confirm.json b/tests/conformance/review/test-integrity-blind-confirm.json index 5bf91a9..edc372f 100644 --- a/tests/conformance/review/test-integrity-blind-confirm.json +++ b/tests/conformance/review/test-integrity-blind-confirm.json @@ -1,5 +1,32 @@ { "schema": "c2-ti/v1", + "description": "Authored blind to the detectors.", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-NET-CONF-P1", @@ -211,7 +238,6 @@ ] } ], - "description": "Authored blind to the detectors.", "label_corrections": [ { "case_id": "C2-TI-GIT-CONF-P3", diff --git a/tests/conformance/review/test-integrity-blind-dev.json b/tests/conformance/review/test-integrity-blind-dev.json index faa309a..fafbfaa 100644 --- a/tests/conformance/review/test-integrity-blind-dev.json +++ b/tests/conformance/review/test-integrity-blind-dev.json @@ -1,5 +1,32 @@ { "schema": "c2-ti/v1", + "description": "Authored blind to the detectors.", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-NET-BLIND-P1", @@ -205,6 +232,5 @@ "source": "import subprocess\nimport tempfile\nfrom pathlib import Path\n\nwith tempfile.TemporaryDirectory(prefix=\"repo-\") as directory:\n repo = Path(directory)\n hooks = repo / \"empty-hooks\"\n hooks.mkdir()\n subprocess.run([\"git\", \"init\", str(repo)], check=True, capture_output=True)\n for key, value in [(\"user.name\", \"Fixture\"), (\"user.email\", \"fixture@example.invalid\"), (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", str(hooks)), (\"remote.origin.url\", \"/var/empty-fixture-remote.git\")]:\n subprocess.run([\"git\", \"-C\", str(repo), \"config\", \"--local\", key, value], check=True, capture_output=True)\n assert list(hooks.iterdir()) == []", "expected_findings": [] } - ], - "description": "Authored blind to the detectors." + ] } diff --git a/tests/conformance/review/test-integrity-blind-final.json b/tests/conformance/review/test-integrity-blind-final.json index 1d548da..5da37d3 100644 --- a/tests/conformance/review/test-integrity-blind-final.json +++ b/tests/conformance/review/test-integrity-blind-final.json @@ -1,6 +1,32 @@ { "schema": "c2-ti/v1", "description": "Final out-of-sample blind holdouts", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-NET-FINAL-P1", diff --git a/tests/conformance/review/test-integrity-blind-gate2.json b/tests/conformance/review/test-integrity-blind-gate2.json index f9e759a..5ca96c3 100644 --- a/tests/conformance/review/test-integrity-blind-gate2.json +++ b/tests/conformance/review/test-integrity-blind-gate2.json @@ -1,6 +1,32 @@ { "schema": "c2-ti/v1", "description": "Gate-2 blind out-of-sample cases, authored without the checklist", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-G2-NET-P1", diff --git a/tests/conformance/review/test-integrity-blind-gate3.json b/tests/conformance/review/test-integrity-blind-gate3.json index bfa0d6d..af20295 100644 --- a/tests/conformance/review/test-integrity-blind-gate3.json +++ b/tests/conformance/review/test-integrity-blind-gate3.json @@ -1,6 +1,32 @@ { "schema": "c2-ti/v1", "description": "Gate-3 blind out-of-sample cases, authored without the checklist or any corpus", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-G3-NET-P1",