diff --git a/scripts/eval-test-integrity-model.py b/scripts/eval-test-integrity-model.py index c34a025..79f594e 100755 --- a/scripts/eval-test-integrity-model.py +++ b/scripts/eval-test-integrity-model.py @@ -182,7 +182,6 @@ def score(args): fail("unexpected files in %s: %s" % (response_dir, ", ".join(extra))) predictions = [] malformed = [] - details = [] for response_dir in response_dirs: batch = {} for case in cases: @@ -193,57 +192,90 @@ def score(args): else: batch[case["id"]] = findings predictions.append(batch) + # Determinism is judged on detected seed ids (issue #28): wording and extra + # findings do not count here. Extra findings still lower each repeat's own + # precision and false-positive counts below. repeat_mismatch = [] if args.repeat > 1: for case in cases: - sets = [None if batch[case["id"]] is None else sorted(set(f["integrity_class"] for f in batch[case["id"]])) for batch in predictions] - if any(item != sets[0] for item in sets[1:]): + if not case["expected_findings"]: + continue + detected = [None if batch[case["id"]] is None else any(f["integrity_class"] == case["class"] for f in batch[case["id"]]) for batch in predictions] + if any(item != detected[0] for item in detected[1:]): repeat_mismatch.append(case["id"]) - tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0 - per_class = {} - for class_id, class_severity in sorted(classes.items()): - per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0} - for case in cases: - expected = bool(case["expected_findings"]) - class_id = case["class"] - class_severity = classes.get(class_id) - if expected: - expected_positive += 1 - per_class.setdefault(class_id, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["expected"] += 1 - if class_severity in ("P0", "P1"): - p01_expected += 1 - findings = predictions[0][case["id"]] - if findings is None: - details.append({"case_id": case["id"], "status": "malformed-or-missing"}) - continue - predicted = set(f["integrity_class"] for f in findings) - predicted_count += len(findings) - if expected and class_id in predicted: - tp += 1 - per_class[class_id]["true_positive"] += 1 - if class_severity in ("P0", "P1"): - p01_tp += 1 - if any(f["integrity_class"] == class_id and f["severity"] == class_severity for f in findings): - severity_match += 1 - per_class[class_id]["severity_agreement"] += 1 - for f in findings: - pred_class = f["integrity_class"] - correct = expected and pred_class == class_id - if not correct: - fp += 1 - per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1 - if not expected and f["severity"] in ("P0", "P1"): - p01_negative_fp += 1 - if pred_class not in classes: - # Unknown class identifiers are malformed schema values. - malformed.append({"case_id": case["id"], "reason": "unknown class " + pred_class}) - details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)}) - precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0) - recall = float(tp) / expected_positive if expected_positive else 1.0 - p01_recall = float(p01_tp) / p01_expected if p01_expected else 1.0 - report = {"positive_recall": recall, "p0_p1_recall": p01_recall, "precision": precision, "clean_negative_p0_p1_false_positives": p01_negative_fp, "malformed_responses": malformed, "repeat_mismatches": repeat_mismatch, "severity_agreement_count": severity_match, "predicted_finding_count": predicted_count, "true_positive_count": tp, "false_positive_count": fp, "per_class": per_class, "cases": details} + + def evaluate(batch): + tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0 + per_class = {} + details = [] + for class_id in sorted(classes): + per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0} + for case in cases: + expected = bool(case["expected_findings"]) + class_id = case["class"] + class_severity = classes.get(class_id) + if expected: + expected_positive += 1 + per_class.setdefault(class_id, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["expected"] += 1 + if class_severity in ("P0", "P1"): + p01_expected += 1 + findings = batch[case["id"]] + if findings is None: + details.append({"case_id": case["id"], "status": "malformed-or-missing"}) + continue + predicted = set(f["integrity_class"] for f in findings) + predicted_count += len(findings) + if expected and class_id in predicted: + tp += 1 + per_class[class_id]["true_positive"] += 1 + if class_severity in ("P0", "P1"): + p01_tp += 1 + if any(f["integrity_class"] == class_id and f["severity"] == class_severity for f in findings): + severity_match += 1 + per_class[class_id]["severity_agreement"] += 1 + for f in findings: + pred_class = f["integrity_class"] + correct = expected and pred_class == class_id + if not correct: + fp += 1 + per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1 + if not expected and f["severity"] in ("P0", "P1"): + p01_negative_fp += 1 + details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)}) + precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0) + return { + "positive_recall": float(tp) / expected_positive if expected_positive else 1.0, + "p0_p1_recall": float(p01_tp) / p01_expected if p01_expected else 1.0, + "precision": precision, + "clean_negative_p0_p1_false_positives": p01_negative_fp, + "severity_agreement_count": severity_match, + "predicted_finding_count": predicted_count, + "true_positive_count": tp, + "false_positive_count": fp, + "per_class": per_class, + "cases": details, + } + + repeats = [evaluate(batch) for batch in predictions] + # The gate must hold in every repeat. The headline is the repeat with the + # worst precision, so its per-class figures, case list and counts agree with + # the headline precision instead of describing another repeat. + headline = min(range(len(repeats)), key=lambda index: (repeats[index]["precision"], -repeats[index]["clean_negative_p0_p1_false_positives"], -repeats[index]["false_positive_count"])) + report = dict(repeats[headline]) + report["headline_repeat"] = headline + 1 + # Worst figures are reported separately so every headline count keeps + # describing the headline repeat. + report["worst"] = { + "positive_recall": min(item["positive_recall"] for item in repeats), + "p0_p1_recall": min(item["p0_p1_recall"] for item in repeats), + "precision": min(item["precision"] for item in repeats), + "clean_negative_p0_p1_false_positives": max(item["clean_negative_p0_p1_false_positives"] for item in repeats), + } + report = dict(report, malformed_responses=malformed, repeat_mismatches=repeat_mismatch) + report["repeats"] = [{key: item[key] for key in ("positive_recall", "p0_p1_recall", "precision", "clean_negative_p0_p1_false_positives", "false_positive_count", "true_positive_count")} for item in repeats] print(json.dumps(report, indent=2, sort_keys=True)) - if p01_recall < 1.0 or precision < 0.90 or p01_negative_fp or malformed or repeat_mismatch: + worst = report["worst"] + if worst["p0_p1_recall"] < 1.0 or worst["precision"] < 0.90 or worst["clean_negative_p0_p1_false_positives"] or malformed or repeat_mismatch: raise SystemExit(1) diff --git a/scripts/test-eval-test-integrity-model.sh b/scripts/test-eval-test-integrity-model.sh index daed51a..3b52b35 100755 --- a/scripts/test-eval-test-integrity-model.sh +++ b/scripts/test-eval-test-integrity-model.sh @@ -152,15 +152,42 @@ with tempfile.TemporaryDirectory(prefix="c2ti-model-test-") as temporary: cross_report = json.loads(cross_result.stdout) assert cross_report["false_positive_count"] == 1 - repeat_a = base / "repeat-a" - repeat_b = base / "repeat-b" - repeat_a.mkdir(); repeat_b.mkdir() - for directory in (repeat_a, repeat_b): - (directory / "CASE-SECRET-CLEAN.json").write_text('{"findings": []}', encoding="utf-8") - (repeat_a / "CASE-SECRET-1.json").write_text('{"findings": []}', encoding="utf-8") - (repeat_b / "CASE-SECRET-1.json").write_text(json.dumps({"findings": [{"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "network call"}]}), encoding="utf-8") - repeat_result = call(["score", "--plan", str(plan_path), "--responses", str(repeat_a), str(repeat_b), "--corpus", str(corpus), "--repeat", "2"], expected=1) - assert "repeat_mismatches" in repeat_result.stdout + # Determinism compares detected seed ids, as issue #28 requires; every + # repeat is also scored on its own. + def write_repeat(name, seed_findings, clean_findings): + directory = base / name + directory.mkdir() + (directory / "CASE-SECRET-1.json").write_text(json.dumps({"findings": seed_findings}), encoding="utf-8") + (directory / "CASE-SECRET-CLEAN.json").write_text(json.dumps({"findings": clean_findings}), encoding="utf-8") + return directory + hit = {"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "network call"} + noise = {"integrity_class": "assertion-specificity", "severity": "P2", "line": 1, "evidence": "noise"} + clean_fp = {"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "false alarm"} + # A seed detected in one repeat and missed in another is a mismatch. + flip = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("flip-a", [hit], [])), str(write_repeat("flip-b", [], [])), "--corpus", str(corpus), "--repeat", "2"], expected=1) + assert json.loads(flip.stdout)["repeat_mismatches"] == ["CASE-SECRET-1"], flip.stdout + # An extra non-seed class varying across repeats is not a seed mismatch; + # it is counted against that repeat's precision instead. + extra_class = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("noise-a", [hit], [])), str(write_repeat("noise-b", [hit, noise], [])), "--corpus", str(corpus), "--repeat", "2"], expected=1) + extra_report = json.loads(extra_class.stdout) + assert extra_report["repeat_mismatches"] == [], extra_class.stdout + assert [item["precision"] for item in extra_report["repeats"]] == [1.0, 0.5], extra_class.stdout + # The headline describes the same repeat as the headline precision. + assert extra_report["headline_repeat"] == 2, extra_class.stdout + assert extra_report["false_positive_count"] == 1 and extra_report["true_positive_count"] == 1, extra_class.stdout + # Every headline figure describes the headline repeat; worst figures are separate. + assert extra_report["positive_recall"] == extra_report["repeats"][1]["positive_recall"], extra_class.stdout + assert extra_report["worst"]["precision"] == 0.5, extra_class.stdout + # A clean-case P0/P1 false positive in a later repeat fails the gate. + late_fp = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("late-a", [hit], [])), str(write_repeat("late-b", [hit], [clean_fp])), "--corpus", str(corpus), "--repeat", "2"], expected=1) + late_report = json.loads(late_fp.stdout) + assert late_report["repeat_mismatches"] == [], late_fp.stdout + assert [item["clean_negative_p0_p1_false_positives"] for item in late_report["repeats"]] == [0, 1], late_fp.stdout + assert late_report["worst"]["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout + assert late_report["headline_repeat"] == 2, late_fp.stdout + assert late_report["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout + # Identical passing repeats pass. + call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("same-a", [hit], [])), str(write_repeat("same-b", [hit], [])), "--corpus", str(corpus), "--repeat", "2"]) # The same directory cannot stand in for several repeats. call(["score", "--plan", str(plan_path), "--responses", str(good), str(good), "--corpus", str(corpus), "--repeat", "2"], expected=1) diff --git a/skills/reviewing/references/test-integrity.md b/skills/reviewing/references/test-integrity.md index b567d53..4d2b430 100644 --- a/skills/reviewing/references/test-integrity.md +++ b/skills/reviewing/references/test-integrity.md @@ -24,7 +24,7 @@ Evidence includes the path expression and the isolation boundary it bypasses. Do ### Verifier fail-open behavior -Check every verifier and assertion adapter for ignored exit status, swallowed exceptions, default-success fallbacks, unchecked `check=False` results, and missing or unknown verdict handling. A malformed, missing, or infrastructure-error result must fail closed rather than become `clean`, `pass`, or an empty finding list. Severity is P0 when the path authorizes a release or mutation and P1 otherwise. +Check every verifier and assertion adapter for ignored exit status, swallowed exceptions, default-success fallbacks, unchecked `check=False` results, and missing or unknown verdict handling. A command substitution whose failure becomes empty output is one of these: `test -z "$(find ...)"`, `[ -z "$(tool ...)" ]`, or a pipeline whose exit status is not checked reports success when the command itself is missing, fails, or errors. A malformed, missing, or infrastructure-error result must fail closed rather than become `clean`, `pass`, or an empty finding list. Severity is P0 when the path authorizes a release or mutation and P1 otherwise. Evidence traces the failure input to the success path. Do not flag a deliberately handled expected rejection whose result is asserted and whose unknown or infrastructure states remain distinct. @@ -38,13 +38,13 @@ Evidence cites the assertion and the untested distinction. Do not flag a deliber Check whether tests read host `HOME`, `PATH`, `TMPDIR`, locale, timezone, credentials, Git configuration outside fixture repositories, network proxy variables, or tool-specific configuration without setting or clearing them in the fixture. Flag environment reads that change the verdict or target and are not represented in the fixture's declared inputs. Severity is P1 when host state can change authorization or target selection and P2 otherwise. -Evidence names the environment key or implicit process state and the missing fixture declaration. Do not flag an explicitly supplied value that the fixture records and verifies. +Evidence names the environment key or implicit process state and the missing fixture declaration. A read of host `HOME` or `PATH` that the snippet does not first set or clear is a finding even when the snippet shown only stores or prints the value, because the fixture has not declared the input. The temporary root is an exception: `mktemp`, `tempfile.gettempdir()`, `tempfile.TemporaryDirectory()`, and the process temporary directory are judged only under Shared temporary directories, and the host `TMPDIR`, `TEMP`, or `TMP` they consult is not a separate Hidden environment dependency unless the snippet uses the value for something else. Do not flag an explicitly supplied value that the fixture records and verifies. ### Fixture Git signing/hooks Check every temporary Git repository for explicit identity, signing policy, hooks path, and remote configuration. A fixture must not inherit host signing keys, global hooks, credential helpers, or remotes. Flag missing `commit.gpgsign=false` when the fixture must commit without host keys, missing empty `core.hooksPath`, and use of an inherited global configuration. Severity is P1 when host code can execute or a commit cannot be reproduced; otherwise P2. -Evidence cites the Git setup and the inherited state that remains possible. Do not flag a fixture that sets all required keys explicitly and proves its hooks directory is empty. +Evidence cites the Git setup and the inherited state that remains possible. Do not flag a fixture that sets identity and `commit.gpgsign=false` locally and takes one of these hook-isolation forms: an empty `core.hooksPath`, a `core.hooksPath` directory the fixture created and does not populate, or `core.hooksPath=/dev/null`. Those keys isolate signing and hooks only. Other inherited Git configuration can still execute host code, so do not describe local keys as complete isolation: a global `core.fsmonitor`, `core.attributesFile` with a `filter..clean` command, or a global `init.templateDir` remains reachable. Flag that residual only when host code can execute through it, and treat a fixture that must exclude host Git configuration entirely as requiring `GIT_CONFIG_GLOBAL` and `GIT_CONFIG_NOSYSTEM`. ### Class boundaries @@ -53,8 +53,9 @@ Report each defect once, under the single class that owns it: - Git identity, signing, hooks, credential helpers, remote settings, and inherited global or system Git configuration of a fixture repository belong to Fixture Git signing/hooks. Do not also report that configuration as a hidden environment dependency. A Git command that actually contacts an external remote, such as `clone`, `fetch`, or `push`, is a separate Network isolation defect. - A configured remote, URL, or host name that the code never contacts is not a network defect. Network isolation needs a call or command that can reach an external host. - An ignored exit status, swallowed exception, or default-success value belongs to Verifier fail-open behavior. Report Assertion specificity only for a test assertion that can pass without the required behavior. -- Hidden environment dependencies covers host state other than Git configuration that the snippet reads or that selects its target or verdict. Running a tool or `git` binary from `PATH` is not by itself an environment defect. +- Hidden environment dependencies covers host state other than Git configuration that the snippet reads or that selects its target or verdict. Running a tool or `git` binary from `PATH` is not by itself an environment defect. Package-source settings such as `PIP_INDEX_URL` or `PIP_CONFIG_FILE` inherited by an install command belong to Network isolation when that command can reach an external index, not to this class. - Judge only the text shown. Do not report a class because unseen helper code might be unsafe. +- Report a missing check in a complete test when the contract needs one: a test whose body never asserts or verifies the behavior it names is an Assertion specificity finding, and so is a test that inspects a result it never checks. A snippet is often a fragment written to show a different defect; when another class owns that defect, do not also report Assertion specificity merely because the fragment has no assertion. ## Per-run evidence record @@ -76,7 +77,7 @@ Run `bash scripts/test-review-test-integrity.sh seeded` to exercise seed and hol ## Model release gate -The release gate scores the real model reviewer against four corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, and `tests/conformance/review/test-integrity-blind-final.json`. The blind corpora were authored without access to the detectors. The final corpus was written after the class boundaries were fixed and was not used to change the checklist, so it is the out-of-sample measurement; two independent judges agreed with all of its labels. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. +The release gate scores the real model reviewer against six corpora: `tests/conformance/review/test-integrity-corpus.json`, `tests/conformance/review/test-integrity-blind-dev.json`, `tests/conformance/review/test-integrity-blind-confirm.json`, `tests/conformance/review/test-integrity-blind-final.json`, `tests/conformance/review/test-integrity-blind-gate2.json`, and `tests/conformance/review/test-integrity-blind-gate3.json`. The blind corpora were authored without access to the detectors. The first five were used to tune or correct the checklist, so they are regression data, not out-of-sample evidence. `test-integrity-blind-gate3.json` is the out-of-sample corpus: it was written after the checklist was fixed, by an author who had only the class descriptions. Two independent judges labeled its sources without seeing the labels and agreed with 35 of its 36; the one dispute is recorded under `label_audit`. Report its figures separately with a plan and score run containing only that corpus; every corpus declares the six class severities, so a single-corpus run scores correctly. `score` itself has no per-corpus breakdown. Keep `scripts/test-review-test-integrity.sh` as an offline deterministic-regex regression check; its result is not the model release verdict. First create the exact request plan without invoking a model: @@ -86,6 +87,8 @@ python3 scripts/eval-test-integrity-model.py plan \ --corpus tests/conformance/review/test-integrity-blind-dev.json \ --corpus tests/conformance/review/test-integrity-blind-confirm.json \ --corpus tests/conformance/review/test-integrity-blind-final.json \ + --corpus tests/conformance/review/test-integrity-blind-gate2.json \ + --corpus tests/conformance/review/test-integrity-blind-gate3.json \ --out /tmp/c2ti-model-plan shasum -a 256 /tmp/c2ti-model-plan/plan.json ``` @@ -103,4 +106,4 @@ python3 scripts/eval-test-integrity-model.py run \ --approval ``` -For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all four `--corpus` arguments to `score` as in `plan`. The gate requires 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, no malformed or missing responses, and identical predicted class-id sets in every repeat. The report also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. +For one run, score with `--responses /tmp/c2ti-run-1`. For the required three-run determinism check, run three times into three different directories and use `--responses --repeat 3`. Supply all six `--corpus` arguments to `score` as in `plan`. The gate requires, in every repeat, 100% P0/P1 recall, at least 90% precision, zero P0/P1 false positives on negative/clean cases, and no malformed or missing responses. Determinism requires the same set of detected seed ids in every repeat; free-text wording and extra findings do not count toward it, but extra findings still lower each repeat's precision. The report shows the worst repeat as its headline figures, lists each repeat under `repeats`, and also includes per-class results and severity agreement; disagreement is reported for review. Planning, scoring, and the fixture tests are offline; only the explicitly approved `run` subcommand makes model calls. diff --git a/tests/conformance/review/test-integrity-blind-confirm.json b/tests/conformance/review/test-integrity-blind-confirm.json index fced708..edc372f 100644 --- a/tests/conformance/review/test-integrity-blind-confirm.json +++ b/tests/conformance/review/test-integrity-blind-confirm.json @@ -1,5 +1,32 @@ { "schema": "c2-ti/v1", + "description": "Authored blind to the detectors.", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-NET-CONF-P1", @@ -198,13 +225,26 @@ ] }, { - "id": "C2-TI-GIT-CONF-N1", - "class": "fixture-git-signing-hooks", + "id": "C2-TI-GIT-CONF-P3", + "class": "verifier-fail-open", "kind": "holdout", "file": "fixture_test.sh", "source": "#!/bin/sh\nrepo=$(mktemp -d)\nmkdir \"$repo/hooks\"\ngit -C \"$repo\" init\ngit -C \"$repo\" config user.name Tester; git -C \"$repo\" config user.email tester@example.invalid\ngit -C \"$repo\" config commit.gpgsign false; git -C \"$repo\" config core.hooksPath \"$repo/hooks\"\ntest -z \"$(find \"$repo/hooks\" -mindepth 1 -print -quit)\"\ngit -C \"$repo\" config remote.origin.url /dev/null\ngit -C \"$repo\" commit --allow-empty -m fixture", - "expected_findings": [] + "expected_findings": [ + { + "seed_id": "C2-TI-GIT-CONF-P3", + "severity": "P1" + } + ] } ], - "description": "Authored blind to the detectors." + "label_corrections": [ + { + "case_id": "C2-TI-GIT-CONF-P3", + "was": "C2-TI-GIT-CONF-N1", + "prior_label": "clean", + "corrected_label": "verifier-fail-open", + "evidence": "/bin/sh without set -e; test -z \"$(find dir -mindepth 1 -print -quit)\" exits 0 when find is missing from PATH, and the script ignores that result" + } + ] } diff --git a/tests/conformance/review/test-integrity-blind-dev.json b/tests/conformance/review/test-integrity-blind-dev.json index faa309a..fafbfaa 100644 --- a/tests/conformance/review/test-integrity-blind-dev.json +++ b/tests/conformance/review/test-integrity-blind-dev.json @@ -1,5 +1,32 @@ { "schema": "c2-ti/v1", + "description": "Authored blind to the detectors.", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-NET-BLIND-P1", @@ -205,6 +232,5 @@ "source": "import subprocess\nimport tempfile\nfrom pathlib import Path\n\nwith tempfile.TemporaryDirectory(prefix=\"repo-\") as directory:\n repo = Path(directory)\n hooks = repo / \"empty-hooks\"\n hooks.mkdir()\n subprocess.run([\"git\", \"init\", str(repo)], check=True, capture_output=True)\n for key, value in [(\"user.name\", \"Fixture\"), (\"user.email\", \"fixture@example.invalid\"), (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", str(hooks)), (\"remote.origin.url\", \"/var/empty-fixture-remote.git\")]:\n subprocess.run([\"git\", \"-C\", str(repo), \"config\", \"--local\", key, value], check=True, capture_output=True)\n assert list(hooks.iterdir()) == []", "expected_findings": [] } - ], - "description": "Authored blind to the detectors." + ] } diff --git a/tests/conformance/review/test-integrity-blind-final.json b/tests/conformance/review/test-integrity-blind-final.json index e7a3c3f..5da37d3 100644 --- a/tests/conformance/review/test-integrity-blind-final.json +++ b/tests/conformance/review/test-integrity-blind-final.json @@ -1,6 +1,32 @@ { "schema": "c2-ti/v1", "description": "Final out-of-sample blind holdouts", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], "cases": [ { "id": "C2-TI-NET-FINAL-P1", @@ -199,12 +225,26 @@ ] }, { - "id": "C2-TI-GIT-FINAL-N1", - "class": "fixture-git-signing-hooks", + "id": "C2-TI-GIT-FINAL-P3", + "class": "verifier-fail-open", "kind": "holdout", "file": "fixture_test.sh", "source": "#!/usr/bin/env bash\nset -euo pipefail\nroot=$(mktemp -d)\ntrap 'rm -rf \"$root\"' EXIT\nmkdir \"$root/hooks\"\nexport GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1\ngit init -q \"$root/repo\"\ng() { git -C \"$root/repo\" -c user.name=Fixture -c user.email=fx@example.invalid -c commit.gpgsign=false -c core.hooksPath=\"$root/hooks\" \"$@\"; }\nprintf 'x\\n' > \"$root/repo/f\"\ng add f\ng commit -q -m seed\n[ -z \"$(ls -A \"$root/hooks\")\" ] && [ \"$(g log --format=%an:%s)\" = \"Fixture:seed\" ]\n", - "expected_findings": [] + "expected_findings": [ + { + "seed_id": "C2-TI-GIT-FINAL-P3", + "severity": "P1" + } + ] + } + ], + "label_corrections": [ + { + "case_id": "C2-TI-GIT-FINAL-P3", + "was": "C2-TI-GIT-FINAL-N1", + "prior_label": "clean", + "corrected_label": "verifier-fail-open", + "evidence": "[ -z \"$(ls -A dir)\" ] exits 0 when ls is missing from PATH, so the check reports an empty hooks directory without inspecting it" } ] -} \ No newline at end of file +} diff --git a/tests/conformance/review/test-integrity-blind-gate2.json b/tests/conformance/review/test-integrity-blind-gate2.json new file mode 100644 index 0000000..5ca96c3 --- /dev/null +++ b/tests/conformance/review/test-integrity-blind-gate2.json @@ -0,0 +1,377 @@ +{ + "schema": "c2-ti/v1", + "description": "Gate-2 blind out-of-sample cases, authored without the checklist", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], + "cases": [ + { + "id": "C2-TI-G2-NET-P1", + "class": "network-isolation", + "kind": "holdout", + "file": "test_forecast_parse.py", + "source": "import requests\n\nfrom weather.client import parse_forecast\n\n\ndef test_parse_forecast_from_service_response():\n resp = requests.get(\n 'https://api.open-meteo.com/v1/forecast',\n params={'latitude': 52.52, 'longitude': 13.41, 'hourly': 'temperature_2m'},\n timeout=10,\n )\n resp.raise_for_status()\n payload = resp.json()\n forecast = parse_forecast(payload)\n assert forecast.latitude == payload['latitude']\n assert [t for t, _ in forecast.hourly] == payload['hourly']['time']\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-NET-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-NET-P2", + "class": "network-isolation", + "kind": "holdout", + "file": "test_depcheck_version.py", + "source": "import subprocess\nimport sys\n\nfrom depcheck import installed_version\n\n\ndef test_installed_version_reads_dist_metadata(tmp_path):\n target = tmp_path / 'site'\n subprocess.run(\n [sys.executable, '-m', 'pip', 'install', '--quiet', '--target', str(target), 'tomli==2.0.1'],\n check=True,\n )\n assert installed_version('tomli', search_path=[str(target)]) == '2.0.1'\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-NET-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-NET-P3", + "class": "network-isolation", + "kind": "holdout", + "file": "test_tcp_banner.py", + "source": "import re\n\nfrom netprobe import tcp_banner\n\n\ndef test_tcp_banner_reads_smtp_greeting():\n banner = tcp_banner('smtp.gmail.com', 587, timeout=5.0)\n assert re.match(r'^220 \\S+ ESMTP', banner)\n assert not banner.endswith('\\r\\n')\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-NET-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-NET-N1", + "class": "network-isolation", + "kind": "holdout", + "file": "test_weather_client.py", + "source": "import httpx\n\nfrom weather.client import WeatherClient\n\n\ndef test_client_parses_hourly_series():\n def handler(request: httpx.Request) -> httpx.Response:\n assert request.url.path == '/v1/forecast'\n return httpx.Response(\n 200,\n json={'latitude': 52.5, 'hourly': {'time': ['t0', 't1'], 'temperature_2m': [3.0, 4.5]}},\n )\n\n transport = httpx.MockTransport(handler)\n with httpx.Client(transport=transport, base_url='https://api.open-meteo.com') as http:\n forecast = WeatherClient(http).forecast(52.5, 13.4)\n assert forecast.latitude == 52.5\n assert forecast.hourly == [('t0', 3.0), ('t1', 4.5)]\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-NET-N2", + "class": "network-isolation", + "kind": "holdout", + "file": "test_tcp_banner_port.py", + "source": "import socket\nimport threading\n\nfrom netprobe import tcp_banner\n\n\ndef test_tcp_banner_reads_first_line_from_loopback():\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as srv:\n srv.bind(('127.0.0.1', 0))\n srv.listen(1)\n port = srv.getsockname()[1]\n\n def serve():\n conn, _ = srv.accept()\n with conn:\n conn.sendall(b'220 mail.test ESMTP ready\\r\\n')\n\n worker = threading.Thread(target=serve)\n worker.start()\n banner = tcp_banner('127.0.0.1', port, timeout=5.0)\n worker.join(timeout=5)\n assert banner == '220 mail.test ESMTP ready'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-TMP-P1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "test_export_report.py", + "source": "import json\nimport os\n\nfrom exporter import export_report\n\n\ndef test_export_report_writes_json():\n out = '/tmp/report-export.json'\n if os.path.exists(out):\n os.remove(out)\n export_report({'total': 3, 'failed': 1}, out)\n with open(out) as fh:\n data = json.load(fh)\n assert data == {'total': 3, 'failed': 1}\n os.remove(out)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-TMP-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-TMP-P2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "test_thumbnails.py", + "source": "import shutil\nimport tempfile\nfrom pathlib import Path\n\nfrom thumbs import render_thumbnails\n\n\ndef test_render_thumbnails_creates_one_file_per_image(sample_images):\n outdir = Path(tempfile.gettempdir()) / 'thumbs-out'\n shutil.rmtree(outdir, ignore_errors=True)\n outdir.mkdir()\n render_thumbnails(sample_images, outdir, size=(64, 64))\n produced = sorted(p.name for p in outdir.iterdir())\n assert produced == sorted(f'{img.stem}.png' for img in sample_images)\n shutil.rmtree(outdir)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-TMP-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-TMP-P3", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "csvsplit_test.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d /tmp/csvsplit.XXXXXX)\ntrap 'rm -rf /tmp/csvsplit.*' EXIT\nprintf 'id,name\\n1,a\\n2,b\\n3,c\\n' > \"$work/in.csv\"\n./bin/csvsplit --rows 2 --out \"$work\" \"$work/in.csv\"\n[ \"$(cat \"$work/part-1.csv\")\" = \"$(printf 'id,name\\n1,a\\n2,b')\" ] || { echo \"part-1 mismatch\" >&2; exit 1; }\n[ \"$(cat \"$work/part-2.csv\")\" = \"$(printf 'id,name\\n3,c')\" ] || { echo \"part-2 mismatch\" >&2; exit 1; }\necho ok\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-TMP-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-TMP-N1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "test_export_roundtrip.py", + "source": "import json\nimport tempfile\nfrom pathlib import Path\n\nfrom exporter import export_report\n\n\ndef test_export_report_round_trips():\n with tempfile.TemporaryDirectory(prefix='report-') as d:\n out = Path(d) / 'report.json'\n export_report({'total': 3, 'failed': 1}, str(out))\n assert json.loads(out.read_text()) == {'total': 3, 'failed': 1}\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-TMP-N2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "csvsplit_parts.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ncleanup() { rm -rf \"$work\"; }\ntrap cleanup EXIT INT TERM\nprintf 'id,name\\n1,a\\n2,b\\n3,c\\n' > \"$work/in.csv\"\n./bin/csvsplit --rows 2 --out \"$work/parts\" \"$work/in.csv\"\n[ \"$(cat \"$work/parts/part-1.csv\")\" = \"$(printf 'id,name\\n1,a\\n2,b')\" ] || { echo \"part-1 mismatch\" >&2; exit 1; }\n[ \"$(cat \"$work/parts/part-2.csv\")\" = \"$(printf 'id,name\\n3,c')\" ] || { echo \"part-2 mismatch\" >&2; exit 1; }\n[ ! -e \"$work/parts/part-3.csv\" ] || { echo \"unexpected part-3\" >&2; exit 1; }\necho ok\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-VER-P1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "verify_schema.py", + "source": "import json\nimport subprocess\nimport sys\n\n\ndef verify(artifact_dir: str) -> dict:\n try:\n proc = subprocess.run(\n [sys.executable, '-m', 'schema_check', artifact_dir],\n capture_output=True, text=True, timeout=120,\n )\n report = json.loads(proc.stdout)\n except Exception:\n return {'verdict': 'pass', 'violations': []}\n violations = report.get('violations', [])\n return {'verdict': 'fail' if violations else 'pass', 'violations': violations}\n\n\nif __name__ == '__main__':\n result = verify(sys.argv[1])\n print(json.dumps(result))\n sys.exit(0 if result['verdict'] == 'pass' else 1)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-VER-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-VER-P2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "check_lint.sh", + "source": "#!/bin/sh\nreport=$(mktemp)\ntrap 'rm -f \"$report\"' EXIT\n./node_modules/.bin/eslint --format json src > \"$report\" 2>/dev/null\nerrors=$(python3 -c 'import json, sys; print(sum(f[\"errorCount\"] for f in json.load(open(sys.argv[1]))))' \"$report\" 2>/dev/null)\nif [ \"${errors:-0}\" -gt 0 ]; then\n echo \"lint: $errors error(s)\"\n exit 1\nfi\necho \"lint: clean\"\nexit 0\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-VER-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-VER-P3", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "grader_adapter.py", + "source": "from dataclasses import dataclass\n\n\n@dataclass\nclass Outcome:\n passed: bool\n detail: str\n\n\n_FAILING = {'fail', 'error'}\n\n\ndef adapt(raw: dict) -> Outcome:\n status = str(raw.get('status', '')).lower()\n if status in _FAILING:\n return Outcome(False, raw.get('message', status))\n return Outcome(True, raw.get('message', 'ok'))\n\n\ndef assert_grader_passed(raw: dict) -> None:\n outcome = adapt(raw)\n assert outcome.passed, outcome.detail\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-VER-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-VER-N1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "verify_schema_report.py", + "source": "import json\nimport subprocess\nimport sys\n\ndef verify(artifact_dir: str) -> dict:\n try:\n proc = subprocess.run(\n [sys.executable, '-m', 'schema_check', artifact_dir],\n capture_output=True, text=True, timeout=120,\n )\n except subprocess.TimeoutExpired:\n return {'verdict': 'error', 'reason': 'timeout'}\n if proc.returncode not in (0, 1):\n return {'verdict': 'error', 'reason': f'exit {proc.returncode}'}\n try:\n violations = json.loads(proc.stdout)['violations']\n except (json.JSONDecodeError, KeyError, TypeError):\n return {'verdict': 'error', 'reason': 'unparseable report'}\n verdict = 'pass' if violations == [] and proc.returncode == 0 else 'fail'\n return {'verdict': verdict, 'violations': violations}\n\nif __name__ == '__main__':\n result = verify(sys.argv[1])\n print(json.dumps(result))\n sys.exit(0 if result['verdict'] == 'pass' else 1)\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-VER-N2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "grader_outcome_adapter.py", + "source": "from dataclasses import dataclass\n\n\n@dataclass\nclass Outcome:\n passed: bool\n detail: str\n\n\ndef adapt(raw: dict) -> Outcome:\n status = raw.get('status')\n if status == 'pass':\n return Outcome(True, raw.get('message', 'pass'))\n if status in ('fail', 'error'):\n return Outcome(False, raw.get('message', status))\n raise ValueError(f'unrecognized grader status: {status!r}')\n\n\ndef assert_grader_passed(raw: dict) -> None:\n outcome = adapt(raw)\n assert outcome.passed, outcome.detail\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-AST-P1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_invoice_discount.py", + "source": "from invoicing import build_invoice\n\n\ndef test_build_invoice_applies_discount():\n invoice = build_invoice(\n items=[('widget', 2, 1000), ('gadget', 1, 2500)],\n discount_pct=10,\n )\n assert invoice\n assert invoice.total\n assert invoice.lines\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-AST-P1", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G2-AST-P2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "uniqsort_test.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nprintf 'b\\na\\nc\\na\\n' > \"$work/in.txt\"\n./bin/uniqsort \"$work/in.txt\" > \"$work/out.txt\"\n[ -f \"$work/out.txt\" ]\necho \"uniqsort ok\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-AST-P2", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G2-AST-P3", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_cli_version.py", + "source": "import subprocess\nimport sys\n\n\ndef test_cli_reports_version():\n proc = subprocess.run(\n [sys.executable, '-m', 'pkgtool', '--version'],\n capture_output=True,\n text=True,\n )\n assert proc.returncode == 0\n assert '.' in proc.stdout\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-AST-P3", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G2-AST-N1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_cli_errors.py", + "source": "import subprocess\nimport sys\n\n\ndef test_cli_rejects_unknown_flag():\n proc = subprocess.run(\n [sys.executable, '-m', 'pkgtool', '--frobnicate'],\n capture_output=True,\n text=True,\n )\n assert proc.returncode == 2\n assert proc.stdout == ''\n assert proc.stderr.splitlines()[-1] == 'pkgtool: error: unrecognized arguments: --frobnicate'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-AST-N2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "test_invoice_totals.py", + "source": "from decimal import Decimal\n\nfrom invoicing import build_invoice\n\n\ndef test_build_invoice_applies_discount():\n invoice = build_invoice(items=[('widget', 2, Decimal('10.00'))], discount_pct=10)\n assert invoice.subtotal == Decimal('20.00')\n assert invoice.discount == Decimal('2.00')\n assert invoice.total == Decimal('18.00')\n assert [line.sku for line in invoice.lines] == ['widget']\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-ENV-P1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_report_header.py", + "source": "from datetime import datetime\n\nfrom reports.header import render_header\n\n\ndef test_render_header_includes_generation_time():\n generated = datetime.fromtimestamp(1700000000)\n header = render_header(title='Weekly', generated_at=generated)\n assert header == 'Weekly \u2014 generated 2023-11-14 22:13'\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-ENV-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-ENV-P2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_registry_token.py", + "source": "from pathlib import Path\n\nfrom uploader.auth import token_for\n\n\ndef test_token_for_registry_reads_netrc():\n netrc = Path.home() / '.netrc'\n line = next(\n entry for entry in netrc.read_text().splitlines() if 'registry.internal' in entry\n )\n expected = line.split('password', 1)[1].strip()\n assert token_for('registry.internal') == expected\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-ENV-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-ENV-P3", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_title_sorting.py", + "source": "import locale\n\nfrom catalog.sorting import sort_titles\n\n\ndef test_sort_titles_uses_collation():\n locale.setlocale(locale.LC_COLLATE, '')\n titles = ['zebra', '\u00c4pfel', 'apple', 'Birne']\n assert sort_titles(titles) == ['apple', '\u00c4pfel', 'Birne', 'zebra']\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-ENV-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-ENV-N1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_report_header_rendering.py", + "source": "from datetime import datetime, timezone\n\nfrom reports.header import render_header\n\n\ndef test_render_header_includes_generation_time():\n generated = datetime.fromtimestamp(1700000000, tz=timezone.utc)\n header = render_header(title='Weekly', generated_at=generated)\n assert header == 'Weekly \u2014 generated 2023-11-14 22:13 UTC'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-ENV-N2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "test_registry_token_fixture.py", + "source": "from uploader.auth import token_for\n\n\ndef test_token_for_registry_reads_netrc(tmp_path, monkeypatch):\n monkeypatch.setenv('HOME', str(tmp_path))\n monkeypatch.delenv('NETRC', raising=False)\n netrc = tmp_path / '.netrc'\n netrc.write_text('machine registry.internal login ci password s3cr3t-token\\n')\n netrc.chmod(0o600)\n assert token_for('registry.internal') == 's3cr3t-token'\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G2-GIT-P1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "test_changelog_entries.py", + "source": "import subprocess\n\nfrom changelog import collect_entries\n\n\ndef _git(repo, *args):\n subprocess.run(['git', '-C', str(repo), *args], check=True, capture_output=True)\n\n\ndef test_collect_entries_reads_commit_subjects(tmp_path):\n repo = tmp_path / 'repo'\n repo.mkdir()\n _git(repo, 'init', '-q')\n _git(repo, 'config', 'user.name', 'Fixture')\n _git(repo, 'config', 'user.email', 'fixture@test.local')\n (repo / 'a.txt').write_text('a\\n')\n _git(repo, 'add', 'a.txt')\n _git(repo, 'commit', '-q', '-m', 'feat: add a')\n assert collect_entries(repo) == ['feat: add a']\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-P2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "release_notes_test.sh", + "source": "#!/bin/sh\nset -eu\nrepo=$(mktemp -d)\ntrap 'rm -rf \"$repo\"' EXIT\ngit init -q \"$repo\"\nprintf 'one\\n' > \"$repo/f\"\ngit -C \"$repo\" add f\ngit -C \"$repo\" -c user.name=Fixture -c user.email=fixture@test.local commit -q -m 'feat: one'\nprintf 'two\\n' >> \"$repo/f\"\ngit -C \"$repo\" -c user.name=Fixture -c user.email=fixture@test.local commit -q -am 'fix: two'\nexpected=$(printf '%s\\n' '- fix: two' '- feat: one')\nactual=$(./bin/release-notes \"$repo\")\n[ \"$actual\" = \"$expected\" ] || { printf 'got:\\n%s\\n' \"$actual\" >&2; exit 1; }\necho ok\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-P3", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "test_tag_release.py", + "source": "import subprocess\n\nfrom releaser import tag_release\n\n\ndef test_tag_release_creates_annotated_tag(tmp_path):\n repo = tmp_path / 'r'\n repo.mkdir()\n\n def run(*args):\n return subprocess.run(['git', *args], cwd=repo, check=True, capture_output=True, text=True)\n\n run('init', '-q')\n run('config', 'user.name', 'CI Fixture')\n run('config', 'user.email', 'ci@fixture.test')\n run('config', 'commit.gpgsign', 'false')\n (repo / 'VERSION').write_text('1.4.0\\n')\n run('add', 'VERSION')\n run('commit', '-q', '-m', 'release 1.4.0')\n tag_release(repo, '1.4.0')\n out = run('for-each-ref', '--format=%(objecttype) %(refname:short)', 'refs/tags')\n assert out.stdout.strip() == 'tag v1.4.0'\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-P4", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "test_mirror_branch.py", + "source": "import subprocess\n\nfrom sync.mirror import mirror_branch\n\n\ndef _git(cwd, *args):\n return subprocess.run(['git', *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef test_mirror_branch_pushes_to_local_remote(tmp_path):\n hooks = tmp_path / 'no-hooks'\n hooks.mkdir()\n origin = tmp_path / 'origin.git'\n _git(tmp_path, 'init', '-q', '--bare', str(origin))\n work = tmp_path / 'work'\n _git(tmp_path, 'init', '-q', '-b', 'main', str(work))\n for key, value in [('user.name', 'Fixture'), ('user.email', 'fixture@test.local'), ('commit.gpgsign', 'false'), ('tag.gpgsign', 'false'), ('core.hooksPath', str(hooks))]:\n _git(work, 'config', key, value)\n _git(work, 'remote', 'add', 'origin', str(origin))\n (work / 'x.txt').write_text('x\\n')\n _git(work, 'add', 'x.txt')\n _git(work, 'commit', '-q', '-m', 'add x')\n mirror_branch(work, 'origin', 'main')\n head = _git(work, 'rev-parse', 'HEAD').strip()\n assert _git(origin, 'rev-parse', 'refs/heads/main').strip() == head\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G2-GIT-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G2-GIT-N2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "release_notes_existing.sh", + "source": "#!/bin/sh\nset -eu\nrepo=$(mktemp -d)\ntrap 'rm -rf \"$repo\"' EXIT\ngit init -q \"$repo\"\ngit -C \"$repo\" config user.name 'Fixture'\ngit -C \"$repo\" config user.email 'fixture@test.local'\ngit -C \"$repo\" config commit.gpgsign false\ngit -C \"$repo\" config core.hooksPath /dev/null\nprintf 'one\\n' > \"$repo/f\"\ngit -C \"$repo\" add f\ngit -C \"$repo\" commit -q -m 'feat: one'\nprintf 'two\\n' >> \"$repo/f\"\ngit -C \"$repo\" commit -q -am 'fix: two'\nexpected=$(printf '%s\\n' '- fix: two' '- feat: one')\nactual=$(./bin/release-notes \"$repo\")\n[ \"$actual\" = \"$expected\" ] || { printf 'got:\\n%s\\n' \"$actual\" >&2; exit 1; }\necho ok\n", + "expected_findings": [] + } + ], + "label_corrections": [ + { + "case_id": "C2-TI-G2-GIT-P4", + "was": "C2-TI-G2-GIT-N1", + "prior_label": "clean", + "corrected_label": "fixture-git-signing-hooks", + "evidence": "global init.templateDir with an executable hooks/post-receive, then git init --bare origin, then push: the hook file is installed in origin.git/hooks and runs on push" + } + ], + "file_name_audit": "Case file names avoid safety-bearing words (strict, exact, utc, locale, isolated, loopback, configured) so the payload never hints at the expected verdict." +} diff --git a/tests/conformance/review/test-integrity-blind-gate3.json b/tests/conformance/review/test-integrity-blind-gate3.json new file mode 100644 index 0000000..af20295 --- /dev/null +++ b/tests/conformance/review/test-integrity-blind-gate3.json @@ -0,0 +1,457 @@ +{ + "schema": "c2-ti/v1", + "description": "Gate-3 blind out-of-sample cases, authored without the checklist or any corpus", + "classes": [ + { + "id": "network-isolation", + "severity": "P1" + }, + { + "id": "shared-temporary-directories", + "severity": "P1" + }, + { + "id": "verifier-fail-open", + "severity": "P1" + }, + { + "id": "assertion-specificity", + "severity": "P2" + }, + { + "id": "hidden-environment-dependencies", + "severity": "P1" + }, + { + "id": "fixture-git-signing-hooks", + "severity": "P1" + } + ], + "cases": [ + { + "id": "C2-TI-G3-NET-P1", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/test_fx_convert.py", + "source": "import requests\n\nfrom fxrates.convert import convert_amount\n\n\ndef test_convert_uses_live_rate():\n resp = requests.get(\"https://api.exchangerate.host/latest?base=USD&symbols=EUR\", timeout=10)\n resp.raise_for_status()\n rate = resp.json()[\"rates\"][\"EUR\"]\n result = convert_amount(100, \"USD\", \"EUR\", rate=rate)\n assert result == round(100 * rate, 2)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-P2", + "class": "network-isolation", + "kind": "holdout", + "file": "ci/check_wheel_contents.sh", + "source": "#!/bin/sh\nset -eu\nworkdir=$(mktemp -d)\ntrap 'rm -rf \"$workdir\"' EXIT\npython3 -m venv \"$workdir/venv\"\n\"$workdir/venv/bin/pip\" install --quiet build==1.2.1\n\"$workdir/venv/bin/python\" -m build --wheel --outdir \"$workdir/dist\" .\nwheel=$(ls \"$workdir\"/dist/*.whl)\nunzip -l \"$wheel\" > \"$workdir/listing.txt\"\ngrep -q 'mediakit/__init__.py' \"$workdir/listing.txt\"\ngrep -q 'mediakit/codecs/opus.py' \"$workdir/listing.txt\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-P3", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/test_digest_mailer.py", + "source": "import smtplib\n\nfrom notifier.mailer import build_message\n\n\ndef test_digest_message_is_accepted_by_relay():\n msg = build_message(to=\"ops@example.org\", subject=\"Daily digest\", body=\"3 jobs failed\")\n with smtplib.SMTP(\"smtp.mailgun.org\", 587, timeout=15) as relay:\n relay.ehlo()\n refused = relay.send_message(msg, from_addr=\"noreply@example.org\")\n assert refused == {}\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-P4", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/transcode_opus.sh", + "source": "#!/bin/sh\nset -eu\nscratch=$(mktemp -d)\ntrap 'rm -rf \"$scratch\"' EXIT\ncurl -fsSL -o \"$scratch/sample.flac\" https://samples.example-media.net/audio/sine-440hz.flac\n./bin/transcode --to opus --bitrate 96k \"$scratch/sample.flac\" \"$scratch/out.opus\"\n./bin/probe --field codec \"$scratch/out.opus\" > \"$scratch/codec.txt\"\n./bin/probe --field bitrate \"$scratch/out.opus\" > \"$scratch/bitrate.txt\"\n[ \"$(cat \"$scratch/codec.txt\")\" = \"opus\" ]\n[ \"$(cat \"$scratch/bitrate.txt\")\" = \"96000\" ]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-NET-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-NET-N1", + "class": "network-isolation", + "kind": "holdout", + "file": "tests/test_fx_client.py", + "source": "from fxrates import client\n\n\nclass FakeResponse:\n def __init__(self, payload):\n self._payload = payload\n self.status_code = 200\n\n def json(self):\n return self._payload\n\n\ndef test_fetch_rate_parses_payload(monkeypatch):\n calls = []\n\n def fake_get(url, timeout):\n calls.append(url)\n return FakeResponse({\"rates\": {\"EUR\": 0.9137}})\n\n monkeypatch.setattr(client.requests, \"get\", fake_get)\n assert client.fetch_rate(\"USD\", \"EUR\") == 0.9137\n assert calls == [\"https://api.exchangerate.host/latest?base=USD&symbols=EUR\"]\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-NET-N2", + "class": "network-isolation", + "kind": "holdout", + "file": "ci/install_from_wheelhouse.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\npython3 -m venv \"$work/venv\"\n\"$work/venv/bin/pip\" --isolated install --no-index --find-links ./tests/wheelhouse --quiet ledgerkit==2.4.0\n\"$work/venv/bin/python\" -c 'import ledgerkit; print(ledgerkit.__version__)' > \"$work/version.txt\"\n[ \"$(cat \"$work/version.txt\")\" = \"2.4.0\" ]\n\"$work/venv/bin/ledgerkit\" balance --input ./tests/data/journal.csv > \"$work/balance.txt\"\ndiff -u ./tests/data/expected_balance.txt \"$work/balance.txt\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-TMP-P1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/test_quarterly_export.py", + "source": "import csv\nimport os\n\nfrom reports.export import export_quarterly\n\n\ndef test_export_writes_header_and_rows():\n out_path = \"/tmp/quarterly_export.csv\"\n export_quarterly(year=2024, quarter=2, destination=out_path)\n with open(out_path, newline=\"\") as fh:\n rows = list(csv.reader(fh))\n os.remove(out_path)\n assert rows[0] == [\"region\", \"revenue\", \"currency\"]\n assert rows[1:] == [[\"EMEA\", \"125000.00\", \"EUR\"], [\"APAC\", \"98000.50\", \"USD\"]]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-P2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/lintcheck_bad_config.sh", + "source": "#!/bin/sh\nset -eu\nout=/tmp/lintcheck-output\nrm -rf \"$out\"\nmkdir -p \"$out\"\n./bin/lintcheck --format json --output \"$out/report.json\" tests/fixtures/bad_config.yaml || status=$?\n[ \"${status:-0}\" -eq 3 ]\npython3 - \"$out/report.json\" <<'EOF'\nimport json, sys\nreport = json.load(open(sys.argv[1]))\nassert [f[\"rule\"] for f in report[\"findings\"]] == [\"duplicate-key\", \"unknown-field\"], report\nEOF\nrm -rf \"$out\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-P3", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/test_dedupe_partitions.py", + "source": "import os\nimport shutil\nimport tempfile\n\nfrom pipeline.stages import dedupe_partitions\n\n\ndef test_dedupe_removes_duplicate_partitions():\n root = os.path.join(tempfile.gettempdir(), \"dedupe-partitions\")\n shutil.rmtree(root, ignore_errors=True)\n os.makedirs(os.path.join(root, \"day=2024-01-01\"))\n os.makedirs(os.path.join(root, \"day=2024-01-01.copy\"))\n os.makedirs(os.path.join(root, \"day=2024-01-02\"))\n removed = dedupe_partitions(root)\n assert removed == [\"day=2024-01-01.copy\"]\n assert sorted(os.listdir(root)) == [\"day=2024-01-01\", \"day=2024-01-02\"]\n shutil.rmtree(root)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-P4", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/thumbnail_resize.sh", + "source": "#!/bin/sh\nset -eu\ndir=$(mktemp -d /tmp/thumbs-XXXXXX)\ncleanup() { rm -rf /tmp/thumbs-*; }\ntrap cleanup EXIT\ncp tests/media/landscape.jpg \"$dir/in.jpg\"\n./bin/thumbnail --width 320 \"$dir/in.jpg\" \"$dir/out.jpg\"\n./bin/imginfo --format '%wx%h' \"$dir/out.jpg\" > \"$dir/size.txt\"\n[ \"$(cat \"$dir/size.txt\")\" = \"320x180\" ]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-TMP-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-TMP-N1", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/test_layered_config.py", + "source": "import os\nimport tempfile\nimport unittest\n\nfrom configparse.loader import load_layered\n\n\nclass LayeredLoadTest(unittest.TestCase):\n def test_override_file_wins(self):\n with tempfile.TemporaryDirectory(prefix=\"layered-\") as root:\n base = os.path.join(root, \"base.toml\")\n override = os.path.join(root, \"override.toml\")\n with open(base, \"w\") as fh:\n fh.write(\"[server]\\nport = 8080\\nworkers = 2\\n\")\n with open(override, \"w\") as fh:\n fh.write(\"[server]\\nworkers = 8\\n\")\n cfg = load_layered([base, override])\n self.assertEqual(cfg[\"server\"], {\"port\": 8080, \"workers\": 8})\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-TMP-N2", + "class": "shared-temporary-directories", + "kind": "holdout", + "file": "tests/logsift_level_filter.sh", + "source": "#!/bin/sh\nset -eu\nscratch=$(mktemp -d)\ntrap 'rm -rf \"$scratch\"' EXIT\nmkdir -p \"$scratch/logs\"\nprintf '2024-03-01T10:00:00Z ERROR disk full\\n2024-03-01T10:00:05Z INFO retry\\n' > \"$scratch/logs/app.log\"\n./bin/logsift --level ERROR \"$scratch/logs\" > \"$scratch/out.txt\"\nprintf '2024-03-01T10:00:00Z ERROR disk full\\n' > \"$scratch/expected.txt\"\ncmp \"$scratch/expected.txt\" \"$scratch/out.txt\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-VER-P1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "ci/verify_integration.sh", + "source": "#!/bin/sh\nset -u\nresults_dir=$1\nmkdir -p \"$results_dir\"\npython3 -m pytest -q tests/integration > \"$results_dir/pytest.log\" 2>&1\nfailed=$(grep -Eo '[0-9]+ failed' \"$results_dir/pytest.log\" | cut -d' ' -f1)\nif [ \"${failed:-0}\" -gt 0 ]; then\n echo \"verdict=FAIL\" > \"$results_dir/verdict.txt\"\n exit 1\nfi\necho \"verdict=PASS\" > \"$results_dir/verdict.txt\"\nexit 0\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-P2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "tests/test_invoice_schema.py", + "source": "import json\nimport subprocess\nimport sys\n\n\ndef run_schema_check(payload_path):\n proc = subprocess.run(\n [sys.executable, \"-m\", \"schemacheck\", \"--json\", payload_path],\n capture_output=True,\n text=True,\n )\n try:\n verdict = json.loads(proc.stdout).get(\"verdict\")\n except ValueError:\n verdict = None\n return verdict != \"fail\"\n\n\ndef test_invoice_payload_matches_schema():\n assert run_schema_check(\"tests/fixtures/invoice_v3.json\")\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-P3", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "release/check_signatures.sh", + "source": "#!/bin/sh\nset -eu\nmanifest=dist/release-manifest.json\nlog=$(mktemp)\ntrap 'rm -f \"$log\"' EXIT\n./scripts/verify_signatures.sh \"$manifest\" > \"$log\" 2>&1 || true\nif grep -q 'BAD SIGNATURE' \"$log\"; then\n echo \"signature check: FAILED\"\n exit 1\nfi\necho \"signature check: OK\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-P4", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "tests/test_month_end_ledger.py", + "source": "def check_ledger_balances(ledger_path):\n from finance.ledger import load_ledger\n\n try:\n ledger = load_ledger(ledger_path)\n mismatches = [\n acct for acct in ledger.accounts\n if acct.computed_balance() != acct.reported_balance\n ]\n except Exception:\n mismatches = []\n return {\"status\": \"clean\" if not mismatches else \"dirty\", \"mismatches\": mismatches}\n\n\ndef test_month_end_ledger_is_clean():\n result = check_ledger_balances(\"tests/data/ledger_2024_06.csv\")\n assert result[\"status\"] == \"clean\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-VER-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-VER-N1", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "ci/verify_integration.sh", + "source": "#!/bin/sh\nset -eu\nresults_dir=$1\nmkdir -p \"$results_dir\"\nif python3 -m pytest -q tests/integration > \"$results_dir/pytest.log\" 2>&1; then\n echo \"verdict=PASS\" > \"$results_dir/verdict.txt\"\n exit 0\nfi\necho \"verdict=FAIL\" > \"$results_dir/verdict.txt\"\nexit 1\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-VER-N2", + "class": "verifier-fail-open", + "kind": "holdout", + "file": "tests/test_invoice_schema_check.py", + "source": "import json\nimport subprocess\nimport sys\n\n\ndef run_schema_check(payload_path):\n proc = subprocess.run(\n [sys.executable, \"-m\", \"schemacheck\", \"--json\", payload_path],\n capture_output=True,\n text=True,\n )\n if proc.returncode not in (0, 1):\n raise RuntimeError(f\"schemacheck crashed: {proc.stderr.strip()}\")\n report = json.loads(proc.stdout)\n verdict = report[\"verdict\"]\n if verdict not in (\"pass\", \"fail\"):\n raise RuntimeError(f\"unknown verdict {verdict!r}\")\n return verdict == \"pass\", report[\"errors\"]\n\n\ndef test_invoice_payload_matches_schema():\n ok, errors = run_schema_check(\"tests/fixtures/invoice_v3.json\")\n assert (ok, errors) == (True, [])\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-AST-P1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_media_probe.py", + "source": "from media.probe import probe_file\n\n\ndef test_probe_reports_video_stream_metadata():\n info = probe_file(\"tests/media/clip_1080p_30fps.mp4\")\n video = [s for s in info.streams if s.kind == \"video\"]\n assert video\n assert video[0].width\n assert video[0].height\n assert video[0].frame_rate\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P1", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-P2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/csv2ledger_conversion.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\ncat > \"$work/input.csv\" <<'EOF'\ndate,amount,currency\n2024-01-05,12.50,EUR\n2024-01-06,-3.10,EUR\n2024-01-07,40.00,USD\nEOF\n./bin/csv2ledger --base-currency EUR --rates tests/data/rates.json \"$work/input.csv\" > \"$work/ledger.txt\"\necho \"csv2ledger conversion ok\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P2", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-P3", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_orderctl_validate.py", + "source": "import subprocess\nimport sys\n\n\ndef test_cli_reports_validation_errors():\n proc = subprocess.run(\n [sys.executable, \"-m\", \"orderctl\", \"validate\", \"tests/fixtures/orders_bad.json\"],\n capture_output=True,\n text=True,\n )\n assert proc.returncode == 2\n assert \"error\" in proc.stderr.lower()\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P3", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-P4", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_signup_events.py", + "source": "from webapp.events import EventBus\nfrom webapp.handlers import register_signup_handlers\n\n\ndef test_signup_sends_welcome_email():\n bus = EventBus()\n sent = []\n register_signup_handlers(bus, mailer=sent.append)\n\n def on_sent(event):\n assert event.payload[\"template\"] == \"welcome\"\n assert event.payload[\"to\"] == \"new.user@example.org\"\n\n bus.subscribe(\"email.sent\", on_sent)\n bus.publish(\"user.signed_up\", {\"email\": \"new.user@example.org\"})\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-AST-P4", + "severity": "P2" + } + ] + }, + { + "id": "C2-TI-G3-AST-N1", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/test_media_probe_fields.py", + "source": "from media.probe import probe_file\n\n\ndef test_probe_reports_video_stream_metadata():\n info = probe_file(\"tests/media/clip_1080p_30fps.mp4\")\n video = [s for s in info.streams if s.kind == \"video\"]\n assert len(video) == 1\n assert (video[0].width, video[0].height) == (1920, 1080)\n assert video[0].frame_rate == 30\n assert video[0].codec == \"h264\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-AST-N2", + "class": "assertion-specificity", + "kind": "holdout", + "file": "tests/orderctl_validate_errors.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nstatus=0\n./bin/orderctl validate tests/fixtures/orders_bad.json > \"$work/out.txt\" 2> \"$work/err.txt\" || status=$?\n[ \"$status\" -eq 2 ]\ncat > \"$work/expected_err.txt\" <<'EOF'\norders_bad.json: order 17: quantity must be positive\norders_bad.json: order 23: unknown sku \"ZX-999\"\nEOF\ndiff -u \"$work/expected_err.txt\" \"$work/err.txt\"\n[ ! -s \"$work/out.txt\" ]\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-ENV-P1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_deploy_target.py", + "source": "import os\n\nfrom deploytool.config import resolve_target\n\n\ndef test_resolve_target_defaults_to_staging():\n cfg_path = os.path.expanduser(\"~/.deploytool/config.toml\")\n target = resolve_target(cli_target=None, config_path=cfg_path)\n assert target.name == \"staging\"\n assert target.region == \"eu-west-1\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-P2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_invoice_due_line.py", + "source": "from datetime import datetime\n\nfrom billing.invoice import render_due_line\n\n\ndef test_due_line_uses_local_calendar_day():\n issued = datetime.fromtimestamp(1717200000)\n line = render_due_line(issued, net_days=30)\n assert line == \"Payment due: 2024-07-01\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-P3", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/wordindex_order.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nprintf 'banana\\nApple\\ncherry\\napple\\n' > \"$work/words.txt\"\n./bin/wordindex \"$work/words.txt\" > \"$work/index.txt\"\nprintf 'Apple\\napple\\nbanana\\ncherry\\n' > \"$work/expected.txt\"\nsort \"$work/words.txt\" | uniq > \"$work/sorted.txt\"\ndiff -u \"$work/sorted.txt\" \"$work/index.txt\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-P4", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_warehouse_connect.py", + "source": "import os\n\n\ndef connection_kwargs(host, dbname, user):\n kwargs = {\"host\": host, \"dbname\": dbname, \"user\": user, \"sslmode\": \"disable\"}\n password = os.environ.get(\"PGPASSWORD\")\n if password:\n kwargs[\"password\"] = password\n return kwargs\n\n\ndef test_connection_kwargs_without_password_uses_peer_auth():\n kwargs = connection_kwargs(host=\"localhost\", dbname=\"analytics\", user=\"etl\")\n assert kwargs == {\"host\": \"localhost\", \"dbname\": \"analytics\", \"user\": \"etl\", \"sslmode\": \"disable\"}\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-ENV-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-ENV-N1", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/test_invoice_due_line.py", + "source": "import time\nfrom datetime import datetime\n\nimport pytest\n\nfrom billing.invoice import render_due_line\n\n\n@pytest.fixture\ndef utc_clock(monkeypatch):\n monkeypatch.setenv(\"TZ\", \"UTC\")\n time.tzset()\n yield\n monkeypatch.undo()\n time.tzset()\n\n\ndef test_due_line_uses_local_calendar_day(utc_clock):\n issued = datetime.fromtimestamp(1717200000)\n line = render_due_line(issued, net_days=30)\n assert line == \"Payment due: 2024-07-01\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-ENV-N2", + "class": "hidden-environment-dependencies", + "kind": "holdout", + "file": "tests/wordindex_order.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\nmkdir -p \"$work/home\"\nprintf 'banana\\nApple\\ncherry\\napple\\n' > \"$work/words.txt\"\nLC_ALL=C HOME=\"$work/home\" ./bin/wordindex \"$work/words.txt\" > \"$work/index.txt\"\nprintf 'Apple\\napple\\nbanana\\ncherry\\n' > \"$work/expected.txt\"\ndiff -u \"$work/expected.txt\" \"$work/index.txt\"\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-GIT-P1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/test_changelog_since_tag.py", + "source": "import subprocess\n\nfrom relnotes.changelog import since_last_tag\n\n\ndef test_changelog_lists_commits_since_tag(tmp_path):\n repo = tmp_path / \"proj\"\n repo.mkdir()\n\n def git(*args):\n subprocess.run([\"git\", *args], cwd=repo, check=True, capture_output=True)\n\n git(\"init\", \"-q\")\n (repo / \"a.txt\").write_text(\"one\\n\")\n git(\"add\", \"a.txt\")\n git(\"commit\", \"-q\", \"-m\", \"feat: first\")\n git(\"tag\", \"v1.0.0\")\n (repo / \"b.txt\").write_text(\"two\\n\")\n git(\"add\", \"b.txt\")\n git(\"commit\", \"-q\", \"-m\", \"fix: second\")\n assert since_last_tag(str(repo)) == [\"fix: second\"]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P1", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-P2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/bumpver_minor.sh", + "source": "#!/bin/sh\nset -eu\nwork=$(mktemp -d)\ntrap 'rm -rf \"$work\"' EXIT\ngit init -q \"$work/repo\"\ngit -C \"$work/repo\" config user.name \"Fixture Bot\"\ngit -C \"$work/repo\" config user.email \"fixture@example.invalid\"\ngit -C \"$work/repo\" config commit.gpgsign false\nprintf 'version = \"0.3.1\"\\n' > \"$work/repo/pyproject.toml\"\ngit -C \"$work/repo\" add pyproject.toml\ngit -C \"$work/repo\" commit -q -m \"release 0.3.1\"\n./bin/bumpver --repo \"$work/repo\" minor\n[ \"$(git -C \"$work/repo\" log -1 --format=%s)\" = \"release 0.4.0\" ]\ngrep -qx 'version = \"0.4.0\"' \"$work/repo/pyproject.toml\"\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P2", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-P3", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/test_publish_release.py", + "source": "import subprocess\n\nfrom shipit.publish import publish_release\n\n\ndef git(*args, cwd):\n return subprocess.run([\"git\", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef test_publish_pushes_release_branch(tmp_path):\n remote, work = tmp_path / \"origin.git\", tmp_path / \"work\"\n git(\"init\", \"-q\", \"--bare\", str(remote), cwd=tmp_path)\n git(\"init\", \"-q\", str(work), cwd=tmp_path)\n for key, value in [(\"user.name\", \"CI\"), (\"user.email\", \"ci@example.invalid\"),\n (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", \"\")]:\n git(\"config\", key, value, cwd=work)\n (work / \"VERSION\").write_text(\"2.1.0\\n\")\n git(\"add\", \"VERSION\", cwd=work)\n git(\"commit\", \"-q\", \"-m\", \"2.1.0\", cwd=work)\n git(\"remote\", \"add\", \"origin\", str(remote), cwd=work)\n publish_release(str(work), branch=\"release/2.1\")\n assert git(\"rev-parse\", \"release/2.1\", cwd=remote) == git(\"rev-parse\", \"HEAD\", cwd=work)\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P3", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-P4", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/diffscope_names.sh", + "source": "#!/bin/sh\nset -eu\nfixture=$(mktemp -d)\nout=$(mktemp)\ntrap 'rm -rf \"$fixture\" \"$out\"' EXIT\ngit init -q \"$fixture\"\ngit -C \"$fixture\" config commit.gpgsign false\ngit -C \"$fixture\" config core.hooksPath \"\"\nmkdir -p \"$fixture/src\"\nprintf 'def handler(event):\\n return 200\\n' > \"$fixture/src/app.py\"\ngit -C \"$fixture\" add src/app.py\ngit -C \"$fixture\" commit -q -m \"initial handler\"\nprintf 'def handler(event):\\n return 204\\n' > \"$fixture/src/app.py\"\n./bin/diffscope --repo \"$fixture\" --format names > \"$out\"\n[ \"$(cat \"$out\")\" = \"src/app.py\" ]\n", + "expected_findings": [ + { + "seed_id": "C2-TI-G3-GIT-P4", + "severity": "P1" + } + ] + }, + { + "id": "C2-TI-G3-GIT-N1", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/test_publish_release.py", + "source": "import subprocess\n\nfrom shipit.publish import publish_release\n\n\ndef git(*args, cwd):\n return subprocess.run([\"git\", *args], cwd=cwd, check=True, capture_output=True, text=True).stdout\n\n\ndef init_repo(path, bare=False):\n git(\"init\", \"-q\", *([\"--bare\"] if bare else []), str(path), cwd=path.parent)\n for key, value in [(\"user.name\", \"CI\"), (\"user.email\", \"ci@example.invalid\"),\n (\"commit.gpgsign\", \"false\"), (\"core.hooksPath\", \"\")]:\n git(\"config\", key, value, cwd=path)\n\n\ndef test_publish_pushes_release_branch(tmp_path):\n remote, work = tmp_path / \"origin.git\", tmp_path / \"work\"\n init_repo(remote, bare=True)\n init_repo(work)\n (work / \"VERSION\").write_text(\"2.1.0\\n\")\n git(\"add\", \"VERSION\", cwd=work)\n git(\"commit\", \"-q\", \"-m\", \"2.1.0\", cwd=work)\n git(\"remote\", \"add\", \"origin\", str(remote), cwd=work)\n publish_release(str(work), branch=\"release/2.1\")\n assert git(\"rev-parse\", \"release/2.1\", cwd=remote) == git(\"rev-parse\", \"HEAD\", cwd=work)\n", + "expected_findings": [] + }, + { + "id": "C2-TI-G3-GIT-N2", + "class": "fixture-git-signing-hooks", + "kind": "holdout", + "file": "tests/diffscope_names.sh", + "source": "#!/bin/sh\nset -eu\nfixture=$(mktemp -d)\nout=$(mktemp)\ntrap 'rm -rf \"$fixture\" \"$out\"' EXIT\ngit init -q \"$fixture\"\ngit -C \"$fixture\" config user.name \"Fixture Bot\"\ngit -C \"$fixture\" config user.email \"fixture@example.invalid\"\ngit -C \"$fixture\" config commit.gpgsign false\ngit -C \"$fixture\" config core.hooksPath \"\"\nmkdir -p \"$fixture/src\"\nprintf 'def handler(event):\\n return 200\\n' > \"$fixture/src/app.py\"\ngit -C \"$fixture\" add src/app.py\ngit -C \"$fixture\" commit -q -m \"initial handler\"\nprintf 'def handler(event):\\n return 204\\n' > \"$fixture/src/app.py\"\n./bin/diffscope --repo \"$fixture\" --format names > \"$out\"\n[ \"$(cat \"$out\")\" = \"src/app.py\" ]\n", + "expected_findings": [] + } + ], + "label_audit": { + "author": "independent author, class descriptions only", + "judges": [ + "reviewer agent", + "scout agent" + ], + "agreement": "35 of 36", + "disputes": [ + { + "case_id": "C2-TI-G3-ENV-P3", + "author": "hidden-environment-dependencies", + "judge_extra": "assertion-specificity", + "resolution": "author label kept; 2 of 3 readings agree" + } + ] + }, + "file_name_audit": "Case file names avoid safety-bearing words (strict, exact, utc, locale, isolated, loopback, configured) so the payload never hints at the expected verdict." +}