fix(review): correct test-integrity determinism, class boundaries, and blind labels

scripts/eval-test-integrity-model.py

@@ -182,7 +182,6 @@ def score(args):
             fail("unexpected files in %s: %s" % (response_dir, ", ".join(extra)))
     predictions = []
     malformed = []
-    details = []
     for response_dir in response_dirs:
         batch = {}
         for case in cases:
@@ -193,57 +192,90 @@ def score(args):
             else:
                 batch[case["id"]] = findings
         predictions.append(batch)
+    # Determinism is judged on detected seed ids (issue #28): wording and extra
+    # findings do not count here. Extra findings still lower each repeat's own
+    # precision and false-positive counts below.
     repeat_mismatch = []
     if args.repeat > 1:
         for case in cases:
-            sets = [None if batch[case["id"]] is None else sorted(set(f["integrity_class"] for f in batch[case["id"]])) for batch in predictions]
-            if any(item != sets[0] for item in sets[1:]):
+            if not case["expected_findings"]:
+                continue
+            detected = [None if batch[case["id"]] is None else any(f["integrity_class"] == case["class"] for f in batch[case["id"]]) for batch in predictions]
+            if any(item != detected[0] for item in detected[1:]):
                 repeat_mismatch.append(case["id"])
-    tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0
-    per_class = {}
-    for class_id, class_severity in sorted(classes.items()):
-        per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0}
-    for case in cases:
-        expected = bool(case["expected_findings"])
-        class_id = case["class"]
-        class_severity = classes.get(class_id)
-        if expected:
-            expected_positive += 1
-            per_class.setdefault(class_id, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["expected"] += 1
-            if class_severity in ("P0", "P1"):
-                p01_expected += 1
-        findings = predictions[0][case["id"]]
-        if findings is None:
-            details.append({"case_id": case["id"], "status": "malformed-or-missing"})
-            continue
-        predicted = set(f["integrity_class"] for f in findings)
-        predicted_count += len(findings)
-        if expected and class_id in predicted:
-            tp += 1
-            per_class[class_id]["true_positive"] += 1
-            if class_severity in ("P0", "P1"):
-                p01_tp += 1
-            if any(f["integrity_class"] == class_id and f["severity"] == class_severity for f in findings):
-                severity_match += 1
-                per_class[class_id]["severity_agreement"] += 1
-        for f in findings:
-            pred_class = f["integrity_class"]
-            correct = expected and pred_class == class_id
-            if not correct:
-                fp += 1
-                per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1
-                if not expected and f["severity"] in ("P0", "P1"):
-                    p01_negative_fp += 1
-            if pred_class not in classes:
-                # Unknown class identifiers are malformed schema values.
-                malformed.append({"case_id": case["id"], "reason": "unknown class " + pred_class})
-        details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)})
-    precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0)
-    recall = float(tp) / expected_positive if expected_positive else 1.0
-    p01_recall = float(p01_tp) / p01_expected if p01_expected else 1.0
-    report = {"positive_recall": recall, "p0_p1_recall": p01_recall, "precision": precision, "clean_negative_p0_p1_false_positives": p01_negative_fp, "malformed_responses": malformed, "repeat_mismatches": repeat_mismatch, "severity_agreement_count": severity_match, "predicted_finding_count": predicted_count, "true_positive_count": tp, "false_positive_count": fp, "per_class": per_class, "cases": details}
+
+    def evaluate(batch):
+        tp = fp = expected_positive = p01_tp = p01_expected = p01_negative_fp = severity_match = predicted_count = 0
+        per_class = {}
+        details = []
+        for class_id in sorted(classes):
+            per_class[class_id] = {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0}
+        for case in cases:
+            expected = bool(case["expected_findings"])
+            class_id = case["class"]
+            class_severity = classes.get(class_id)
+            if expected:
+                expected_positive += 1
+                per_class.setdefault(class_id, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["expected"] += 1
+                if class_severity in ("P0", "P1"):
+                    p01_expected += 1
+            findings = batch[case["id"]]
+            if findings is None:
+                details.append({"case_id": case["id"], "status": "malformed-or-missing"})
+                continue
+            predicted = set(f["integrity_class"] for f in findings)
+            predicted_count += len(findings)
+            if expected and class_id in predicted:
+                tp += 1
+                per_class[class_id]["true_positive"] += 1
+                if class_severity in ("P0", "P1"):
+                    p01_tp += 1
+                if any(f["integrity_class"] == class_id and f["severity"] == class_severity for f in findings):
+                    severity_match += 1
+                    per_class[class_id]["severity_agreement"] += 1
+            for f in findings:
+                pred_class = f["integrity_class"]
+                correct = expected and pred_class == class_id
+                if not correct:
+                    fp += 1
+                    per_class.setdefault(pred_class, {"expected": 0, "true_positive": 0, "false_positive": 0, "severity_agreement": 0})["false_positive"] += 1
+                    if not expected and f["severity"] in ("P0", "P1"):
+                        p01_negative_fp += 1
+            details.append({"case_id": case["id"], "expected_class": class_id if expected else None, "predicted_classes": sorted(predicted)})
+        precision = float(tp) / predicted_count if predicted_count else (1.0 if expected_positive == 0 else 0.0)
+        return {
+            "positive_recall": float(tp) / expected_positive if expected_positive else 1.0,
+            "p0_p1_recall": float(p01_tp) / p01_expected if p01_expected else 1.0,
+            "precision": precision,
+            "clean_negative_p0_p1_false_positives": p01_negative_fp,
+            "severity_agreement_count": severity_match,
+            "predicted_finding_count": predicted_count,
+            "true_positive_count": tp,
+            "false_positive_count": fp,
+            "per_class": per_class,
+            "cases": details,
+        }
+
+    repeats = [evaluate(batch) for batch in predictions]
+    # The gate must hold in every repeat. The headline is the repeat with the
+    # worst precision, so its per-class figures, case list and counts agree with
+    # the headline precision instead of describing another repeat.
+    headline = min(range(len(repeats)), key=lambda index: (repeats[index]["precision"], -repeats[index]["clean_negative_p0_p1_false_positives"], -repeats[index]["false_positive_count"]))
+    report = dict(repeats[headline])
+    report["headline_repeat"] = headline + 1
+    # Worst figures are reported separately so every headline count keeps
+    # describing the headline repeat.
+    report["worst"] = {
+        "positive_recall": min(item["positive_recall"] for item in repeats),
+        "p0_p1_recall": min(item["p0_p1_recall"] for item in repeats),
+        "precision": min(item["precision"] for item in repeats),
+        "clean_negative_p0_p1_false_positives": max(item["clean_negative_p0_p1_false_positives"] for item in repeats),
+    }
+    report = dict(report, malformed_responses=malformed, repeat_mismatches=repeat_mismatch)
+    report["repeats"] = [{key: item[key] for key in ("positive_recall", "p0_p1_recall", "precision", "clean_negative_p0_p1_false_positives", "false_positive_count", "true_positive_count")} for item in repeats]
     print(json.dumps(report, indent=2, sort_keys=True))
-    if p01_recall < 1.0 or precision < 0.90 or p01_negative_fp or malformed or repeat_mismatch:
+    worst = report["worst"]
+    if worst["p0_p1_recall"] < 1.0 or worst["precision"] < 0.90 or worst["clean_negative_p0_p1_false_positives"] or malformed or repeat_mismatch:
         raise SystemExit(1)
 
 

scripts/test-eval-test-integrity-model.sh

@@ -152,15 +152,42 @@ with tempfile.TemporaryDirectory(prefix="c2ti-model-test-") as temporary:
     cross_report = json.loads(cross_result.stdout)
     assert cross_report["false_positive_count"] == 1
 
-    repeat_a = base / "repeat-a"
-    repeat_b = base / "repeat-b"
-    repeat_a.mkdir(); repeat_b.mkdir()
-    for directory in (repeat_a, repeat_b):
-        (directory / "CASE-SECRET-CLEAN.json").write_text('{"findings": []}', encoding="utf-8")
-    (repeat_a / "CASE-SECRET-1.json").write_text('{"findings": []}', encoding="utf-8")
-    (repeat_b / "CASE-SECRET-1.json").write_text(json.dumps({"findings": [{"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "network call"}]}), encoding="utf-8")
-    repeat_result = call(["score", "--plan", str(plan_path), "--responses", str(repeat_a), str(repeat_b), "--corpus", str(corpus), "--repeat", "2"], expected=1)
-    assert "repeat_mismatches" in repeat_result.stdout
+    # Determinism compares detected seed ids, as issue #28 requires; every
+    # repeat is also scored on its own.
+    def write_repeat(name, seed_findings, clean_findings):
+        directory = base / name
+        directory.mkdir()
+        (directory / "CASE-SECRET-1.json").write_text(json.dumps({"findings": seed_findings}), encoding="utf-8")
+        (directory / "CASE-SECRET-CLEAN.json").write_text(json.dumps({"findings": clean_findings}), encoding="utf-8")
+        return directory
+    hit = {"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "network call"}
+    noise = {"integrity_class": "assertion-specificity", "severity": "P2", "line": 1, "evidence": "noise"}
+    clean_fp = {"integrity_class": "network-isolation", "severity": "P1", "line": 1, "evidence": "false alarm"}
+    # A seed detected in one repeat and missed in another is a mismatch.
+    flip = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("flip-a", [hit], [])), str(write_repeat("flip-b", [], [])), "--corpus", str(corpus), "--repeat", "2"], expected=1)
+    assert json.loads(flip.stdout)["repeat_mismatches"] == ["CASE-SECRET-1"], flip.stdout
+    # An extra non-seed class varying across repeats is not a seed mismatch;
+    # it is counted against that repeat's precision instead.
+    extra_class = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("noise-a", [hit], [])), str(write_repeat("noise-b", [hit, noise], [])), "--corpus", str(corpus), "--repeat", "2"], expected=1)
+    extra_report = json.loads(extra_class.stdout)
+    assert extra_report["repeat_mismatches"] == [], extra_class.stdout
+    assert [item["precision"] for item in extra_report["repeats"]] == [1.0, 0.5], extra_class.stdout
+    # The headline describes the same repeat as the headline precision.
+    assert extra_report["headline_repeat"] == 2, extra_class.stdout
+    assert extra_report["false_positive_count"] == 1 and extra_report["true_positive_count"] == 1, extra_class.stdout
+    # Every headline figure describes the headline repeat; worst figures are separate.
+    assert extra_report["positive_recall"] == extra_report["repeats"][1]["positive_recall"], extra_class.stdout
+    assert extra_report["worst"]["precision"] == 0.5, extra_class.stdout
+    # A clean-case P0/P1 false positive in a later repeat fails the gate.
+    late_fp = call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("late-a", [hit], [])), str(write_repeat("late-b", [hit], [clean_fp])), "--corpus", str(corpus), "--repeat", "2"], expected=1)
+    late_report = json.loads(late_fp.stdout)
+    assert late_report["repeat_mismatches"] == [], late_fp.stdout
+    assert [item["clean_negative_p0_p1_false_positives"] for item in late_report["repeats"]] == [0, 1], late_fp.stdout
+    assert late_report["worst"]["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout
+    assert late_report["headline_repeat"] == 2, late_fp.stdout
+    assert late_report["clean_negative_p0_p1_false_positives"] == 1, late_fp.stdout
+    # Identical passing repeats pass.
+    call(["score", "--plan", str(plan_path), "--responses", str(write_repeat("same-a", [hit], [])), str(write_repeat("same-b", [hit], [])), "--corpus", str(corpus), "--repeat", "2"])
     # The same directory cannot stand in for several repeats.
     call(["score", "--plan", str(plan_path), "--responses", str(good), str(good), "--corpus", str(corpus), "--repeat", "2"], expected=1)