From 903630599d32d619c5823ba034502e58f7110395 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 14:37:15 +0000 Subject: [PATCH] test(evals): Tier E negative control for the cross-environment parity check Ports the Tier E scenario from the archived PostgreDataMigrationApp repo (commit f902ae5, never in this repo) to the current runner, whose schema query and error handling have changed since: _column_signature now raises on a failed query (issue #45), and returns a table -> columns map. Scenario 01 passes when every environment matches dev. That only means something if a deploy that wrote nothing is visible. Scenario 02, against the real database: - creates a throwaway empty schema beside dev's and requires the real schema query to read it as empty (a failed query is a failure, not an empty schema); - requires comparing dev's real schema with it to report the missing tables, which the original did not check, though it is the stated purpose; - refuses to run when dev is not deployed, since comparing nothing with nothing proves nothing; - always drops the throwaway schema, and never with CASCADE, so it cannot delete something that is not its own. Unit tests (mocked psql) pin each branch, including that the control fails when the comparison is neutralised. Verified against PostgreSQL 16 by neutralising the comparison, planting a non-empty schema under the control's name (it is left alone) and reintroducing the #45 failure mode: each turns the scenario red. README's scenario count (26 -> 27) and the Tier E notes in evals/HANDOFF.md and evals/FAILURE_MODES.md are updated. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VBrxqChRJtxdvSpFhiUWUy --- README.md | 2 +- docs/TEST_CASES.md | 62 +++++++- evals/FAILURE_MODES.md | 3 + evals/HANDOFF.md | 4 +- .../02_empty_schema_is_detected/NOTES.txt | 42 ++++++ .../tier_e/02_empty_schema_is_detected.json | 10 ++ evals/runner.py | 109 ++++++++++++++ tests/test_evals_tier_e_x.py | 133 ++++++++++++++++++ 8 files changed, 362 insertions(+), 3 deletions(-) create mode 100644 evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt create mode 100644 evals/expected/tier_e/02_empty_schema_is_detected.json diff --git a/README.md b/README.md index 8cf1d6a..0fb8cc6 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ bash tests/run_tests.sh dev ### Eval suite ```powershell -python evals\runner.py --tiers p,i,s,e # 26 scenarios: Tier P 23, I 1, S 1, E 1 +python evals\runner.py --tiers p,i,s,e # 27 scenarios: Tier P 23, I 1, S 1, E 2 ``` ### CI diff --git a/docs/TEST_CASES.md b/docs/TEST_CASES.md index 9dd0768..d0af918 100644 --- a/docs/TEST_CASES.md +++ b/docs/TEST_CASES.md @@ -1,7 +1,7 @@ # Test Catalogue > Auto-generated by `scripts/test_catalogue.py` — do not edit by hand. -> 732 tests across 38 files. +> 762 tests across 38 files. To regenerate: `python3 scripts/test_catalogue.py` @@ -745,6 +745,36 @@ To regenerate: `python3 scripts/test_catalogue.py` | `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | +| `TierEEmptySchemaControl.test_passes_when_an_empty_schema_reads_empty_and_the_comparison_flags_it` | unit | — | + +| `TierEEmptySchemaControl.test_fails_when_the_comparison_no_longer_reports_the_missing_tables` | unit | — | + +| `TierEEmptySchemaControl.test_fails_when_an_empty_schema_reports_tables` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_query_on_the_empty_schema_is_not_read_as_no_tables` | unit | — | + +| `TierEEmptySchemaControl.test_refuses_to_run_against_an_empty_reference_and_creates_nothing` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_reference_query_is_reported_and_creates_nothing` | unit | — | + +| `TierEEmptySchemaControl.test_creates_then_drops_the_throwaway_schema` | unit | — | + +| `TierEEmptySchemaControl.test_drops_the_throwaway_schema_even_when_the_check_fails` | unit | — | + +| `TierEEmptySchemaControl.test_never_drops_with_cascade` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_drop_is_reported` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_create_is_reported_and_no_schema_query_is_run_on_it` | unit | — | + +| `TierEEmptySchemaControl.test_an_unknown_reference_env_is_reported` | unit | — | + +| `TierEEmptySchemaControl.test_the_scenario_is_dispatched_by_name` | unit | — | + +| `TierEEmptySchemaControl.test_an_unknown_tier_e_scenario_is_still_an_error` | unit | — | + +| `TierEEmptySchemaControl.test_the_scenario_files_exist_and_agree` | unit | — | + | `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | | `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | @@ -789,6 +819,36 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_genuinely_empty_schema_still_fails` | — | — | +| `test_passes_when_an_empty_schema_reads_empty_and_the_comparison_flags_it` | — | — | + +| `test_fails_when_the_comparison_no_longer_reports_the_missing_tables` | — | — | + +| `test_fails_when_an_empty_schema_reports_tables` | — | — | + +| `test_a_failed_query_on_the_empty_schema_is_not_read_as_no_tables` | — | — | + +| `test_refuses_to_run_against_an_empty_reference_and_creates_nothing` | — | — | + +| `test_a_failed_reference_query_is_reported_and_creates_nothing` | — | — | + +| `test_creates_then_drops_the_throwaway_schema` | — | — | + +| `test_drops_the_throwaway_schema_even_when_the_check_fails` | — | — | + +| `test_never_drops_with_cascade` | — | — | + +| `test_a_failed_drop_is_reported` | — | — | + +| `test_a_failed_create_is_reported_and_no_schema_query_is_run_on_it` | — | — | + +| `test_an_unknown_reference_env_is_reported` | — | — | + +| `test_the_scenario_is_dispatched_by_name` | — | — | + +| `test_an_unknown_tier_e_scenario_is_still_an_error` | — | — | + +| `test_the_scenario_files_exist_and_agree` | — | — | + | `test_placeholders_are_substituted` | — | — | | `test_every_known_table_has_a_substitution` | — | — | diff --git a/evals/FAILURE_MODES.md b/evals/FAILURE_MODES.md index c68bd05..004c6eb 100644 --- a/evals/FAILURE_MODES.md +++ b/evals/FAILURE_MODES.md @@ -79,6 +79,9 @@ Tier S initial scope: only S1. - ~~**Cross-environment structural equivalence** (Dev vs Test vs Staging vs Prod) — Tier E, future.~~ *Now covered as of 2026-09-10:* Tier E scenario 01 compares all four deployed schemas column by column and treats reordered columns as drift. Closes VCRM BR-01. + Scenario 02 is its negative control: against the real database, an empty schema must + read as empty and comparing it with Dev must report the missing tables, so 01 cannot + pass a deploy that wrote nothing. - **Domain-rule deep dives beyond suite 05** — Tier D, future. - **Validator behaviour on >128KB single field** — beyond the current 50KB eval and Python `csv` default field-size assumptions. diff --git a/evals/HANDOFF.md b/evals/HANDOFF.md index 0f2f5f6..59d6924 100644 --- a/evals/HANDOFF.md +++ b/evals/HANDOFF.md @@ -6,7 +6,9 @@ > current state. Affected lines are marked inline. > > - **Tier E** — implemented. `evals/datasets/tier_e/01_envs_have_identical_structure`. -> Runs in the `integration-postgres` CI job. Closes VCRM BR-01. +> Runs in the `integration-postgres` CI job. Closes VCRM BR-01. Scenario +> `02_empty_schema_is_detected` is its negative control: it shows 01 can see a deploy +> that wrote nothing. > - **Tier X** — implemented. `evals/datasets/tier_x/01_sqlite_schema_structure`. > Deliberately **not** in CI per [ADR-0001](../docs/adr/0001-postgresql-first.md) > (PostgreSQL first); run it with `make evals-engines`. diff --git a/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt b/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt new file mode 100644 index 0000000..c4300a1 --- /dev/null +++ b/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt @@ -0,0 +1,42 @@ +Tier E scenario 02 - negative control for scenario 01 +===================================================== + +What it proves +-------------- +Scenario 01 passes when Test, Staging and Prod have the same structure as Dev. +That is only worth anything if "a deploy wrote nothing" is something it can see. +This scenario shows that, against the real database: + +1. A schema with no tables is read as having no tables by the same query + scenario 01 uses (_column_signature), and a failed query is reported as a + failure rather than as "no tables" (the confusion behind issue #45). +2. Comparing Dev's real schema with that empty one reports the missing tables + (_diff_signatures), so scenario 01 would fail on it. + +If either stops being true, scenario 01 could go green while a deploy silently +wrote nothing. + +How +--- +It creates a throwaway empty schema "_te_empty_neg_ctrl" in Dev's database, +reads its signature, compares it with Dev's, and drops it again. The drop never +uses CASCADE: if anything else is found in that schema the scenario fails rather +than deleting what is not its own. + +It refuses to run (fails) if Dev has fewer than min_reference_tables tables, +because comparing nothing with nothing would prove nothing. Scenario 01 deploys +Dev first, so run them together (the default for --tiers e); with --only, deploy +Dev beforehand. + +Relationship to the unit tests +------------------------------ +tests/test_evals_tier_e_x.py covers the same logic with a mocked psql. This is the +same claim against a real server, which is what mocks cannot show. + +Fails rather than skips +----------------------- +No PostgreSQL means a failure, as for scenario 01 (FIXES_APPLIED.md F5). + +Run +--- + python3 evals/runner.py --tiers e --verbose diff --git a/evals/expected/tier_e/02_empty_schema_is_detected.json b/evals/expected/tier_e/02_empty_schema_is_detected.json new file mode 100644 index 0000000..54647df --- /dev/null +++ b/evals/expected/tier_e/02_empty_schema_is_detected.json @@ -0,0 +1,10 @@ +{ + "scenario": "02_empty_schema_is_detected", + "description": "Negative control for scenario 01: a schema with no tables must read as empty, and comparing it with Dev's real schema must report the missing tables, so the parity eval cannot pass a deploy that wrote nothing.", + "reference_env": "dev", + "expected": { + "min_reference_tables": 12, + "empty_schema_tables": 0, + "diff_reports_missing_tables": true + } +} diff --git a/evals/runner.py b/evals/runner.py index ffffc25..4dc5627 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -677,6 +677,9 @@ def run_tier_e_scenario(scenario_dir: Path) -> ScenarioResult: if name == "01_envs_have_identical_structure": return _run_envs_have_identical_structure(result, expected) + if name == "02_empty_schema_is_detected": + return _run_empty_schema_negative_control(result, expected) + result.errors.append("Unknown tier-E scenario: " + name) return result @@ -748,6 +751,112 @@ def _run_envs_have_identical_structure( return result +# A schema that only the negative control below creates and drops. It is never +# dropped with CASCADE: if something else is found in it, the control fails +# rather than deleting what is not its own. +_EMPTY_CONTROL_SCHEMA = "_te_empty_neg_ctrl" + + +def _psql_run(db: str, sql: str) -> "subprocess.CompletedProcess[str]": + """Run one SQL statement on `db`; the caller reads returncode and stderr.""" + return subprocess.run( + ["psql", "-d", db, "-v", "ON_ERROR_STOP=1", "-c", sql], + env=_pg_env(), capture_output=True, text=True, timeout=30, + ) + + +def _run_empty_schema_negative_control( + result: ScenarioResult, expected: Dict[str, Any] +) -> ScenarioResult: + """Negative control for scenario 01: show that it can see "nothing deployed". + + Scenario 01 passes when every environment has the same structure as dev. That + only means something if a schema a deploy left empty is read as empty and the + comparison then reports it. This creates a throwaway empty schema beside dev's + and checks both, against the real database: + + 1. the real schema query returns no tables for it (and a failed query is a + failure, not an empty schema: the confusion behind issue #45); + 2. the real comparison between dev's schema and the empty one reports the + missing tables. + + If either stops being true, scenario 01 could pass while a deploy wrote + nothing. The pure logic is unit-tested with a mocked psql; this is the same + claim made against a real server. + """ + exp = expected.get("expected", {}) + ref_env = expected.get("reference_env", "dev") + target = next((t for t in ENV_TARGETS if t[0] == ref_env), None) + if target is None: + result.errors.append("reference_env " + ref_env + " is not one of the " + "environments in ENV_TARGETS") + return result + _, db, ref_schema = target + + # Without a populated reference the comparison below is "nothing against + # nothing" and would prove nothing, so that is an error, not a pass. + try: + ref = _column_signature(db, ref_schema) + except RuntimeError as exc: + result.errors.append("reference env " + ref_env + ": " + str(exc)) + return result + min_reference = int(exp.get("min_reference_tables", 12)) + if len(ref) < min_reference: + result.errors.append( + "reference env " + ref_env + " has " + str(len(ref)) + " tables, " + "expected >= " + str(min_reference) + ". Deploy it first (scenario 01 " + "does); an empty reference would make this control compare nothing " + "with nothing." + ) + return result + + quoted = '"' + _EMPTY_CONTROL_SCHEMA + '"' + made = _psql_run(db, "DROP SCHEMA IF EXISTS " + quoted + + "; CREATE SCHEMA " + quoted) + if made.returncode != 0: + result.errors.append("could not create the throwaway schema " + + _EMPTY_CONTROL_SCHEMA + " in " + db + ": " + + made.stderr.strip()[-300:]) + return result + + errors: List[str] = [] + try: + try: + empty = _column_signature(db, _EMPTY_CONTROL_SCHEMA) + except RuntimeError as exc: + empty = None + errors.append("the schema query failed on an empty schema, which " + "must not be reported as 'no tables': " + str(exc)) + if empty is not None: + want = int(exp.get("empty_schema_tables", 0)) + diffs = _diff_signatures(ref_env, ref, "empty", empty) + result.actual = { + "reference_tables": len(ref), + "empty_schema_tables": len(empty), + "differences_reported": len(diffs), + } + if len(empty) != want: + errors.append("an empty schema returned " + str(len(empty)) + + " tables, expected " + str(want)) + if exp.get("diff_reports_missing_tables") is True and not any( + " is missing tables present in " in d for d in diffs): + errors.append( + "comparing " + ref_env + " (" + str(len(ref)) + " tables) with " + "an empty schema reported no missing tables, so scenario 01 " + "would pass a deploy that wrote nothing" + ) + finally: + dropped = _psql_run(db, "DROP SCHEMA IF EXISTS " + quoted) + if dropped.returncode != 0: + errors.append("could not drop the throwaway schema " + + _EMPTY_CONTROL_SCHEMA + " (it is left in " + db + + "): " + dropped.stderr.strip()[-300:]) + + result.errors = errors + result.passed = not errors + return result + + # --------------------------------------------------------------------------- # Tier X — Cross-engine structural coverage (VCRM BR-02) # diff --git a/tests/test_evals_tier_e_x.py b/tests/test_evals_tier_e_x.py index 51efa99..337d390 100644 --- a/tests/test_evals_tier_e_x.py +++ b/tests/test_evals_tier_e_x.py @@ -126,6 +126,139 @@ def test_genuinely_empty_schema_still_fails(self): "(deploy may have partially failed)", result.errors) +@pytest.mark.unit +class TierEEmptySchemaControl(unittest.TestCase): + """Scenario 02, the negative control for scenario 01. + + Against a real server it proves scenario 01 can see "a deploy wrote nothing". + Here psql and the schema query are mocked, to pin the control's own logic: + above all that it FAILS when the comparison stops reporting an empty schema, + since a control that cannot fail is worth nothing. + """ + + REF = {"t%d" % i: ["id:uuid:NO"] for i in range(12)} + EXPECTED = { + "reference_env": "dev", + "expected": {"min_reference_tables": 12, "empty_schema_tables": 0, + "diff_reports_missing_tables": True}, + } + + def _run(self, signatures, psql=None, diff=None, expected=None): + """Run the control; returns (result, psql mock, signature mock).""" + ok = subprocess.CompletedProcess([], 0, stdout="", stderr="") + result = runner.ScenarioResult(tier="e", name="02_empty_schema_is_detected") + with patch.object(runner, "_column_signature", side_effect=signatures) as sig, \ + patch.object(runner, "_psql_run", + **({"side_effect": psql} if psql else {"return_value": ok})) as run: + if diff is None: + runner._run_empty_schema_negative_control(result, expected or self.EXPECTED) + else: + with patch.object(runner, "_diff_signatures", return_value=diff): + runner._run_empty_schema_negative_control(result, expected or self.EXPECTED) + return result, run, sig + + @staticmethod + def _sql(run): + return [c.args[1] for c in run.call_args_list] + + def test_passes_when_an_empty_schema_reads_empty_and_the_comparison_flags_it(self): + result, run, _ = self._run([self.REF, {}]) + self.assertTrue(result.passed, result.errors) + self.assertEqual(result.actual["reference_tables"], 12) + self.assertEqual(result.actual["empty_schema_tables"], 0) + self.assertGreaterEqual(result.actual["differences_reported"], 1) + + def test_fails_when_the_comparison_no_longer_reports_the_missing_tables(self): + # The point of a control: neutralise the thing it guards and it must go red. + result, _, _ = self._run([self.REF, {}], diff=[]) + self.assertFalse(result.passed) + self.assertTrue(any("would pass a deploy that wrote nothing" in e + for e in result.errors), result.errors) + + def test_fails_when_an_empty_schema_reports_tables(self): + result, _, _ = self._run([self.REF, {"x": ["id:uuid:NO"]}]) + self.assertFalse(result.passed) + self.assertIn("an empty schema returned 1 tables, expected 0", result.errors) + + def test_a_failed_query_on_the_empty_schema_is_not_read_as_no_tables(self): + result, _, _ = self._run([self.REF, RuntimeError("connection lost")]) + self.assertFalse(result.passed) + self.assertTrue(any("must not be reported as 'no tables'" in e + and "connection lost" in e for e in result.errors), result.errors) + + def test_refuses_to_run_against_an_empty_reference_and_creates_nothing(self): + result, run, _ = self._run([{}]) + self.assertFalse(result.passed) + self.assertTrue(any("Deploy it first" in e for e in result.errors), result.errors) + run.assert_not_called() + + def test_a_failed_reference_query_is_reported_and_creates_nothing(self): + result, run, _ = self._run([RuntimeError("db down")]) + self.assertFalse(result.passed) + self.assertIn("reference env dev: db down", result.errors) + run.assert_not_called() + + def test_creates_then_drops_the_throwaway_schema(self): + _, run, _ = self._run([self.REF, {}]) + sql = self._sql(run) + self.assertEqual(len(sql), 2) + self.assertIn("CREATE SCHEMA", sql[0]) + self.assertTrue(sql[1].startswith("DROP SCHEMA IF EXISTS")) + self.assertIn(runner._EMPTY_CONTROL_SCHEMA, sql[1]) + + def test_drops_the_throwaway_schema_even_when_the_check_fails(self): + _, run, _ = self._run([self.REF, RuntimeError("boom")]) + self.assertTrue(self._sql(run)[-1].startswith("DROP SCHEMA IF EXISTS")) + + def test_never_drops_with_cascade(self): + _, run, _ = self._run([self.REF, {}]) + self.assertFalse(any("CASCADE" in q.upper() for q in self._sql(run)), self._sql(run)) + + def test_a_failed_drop_is_reported(self): + ok = subprocess.CompletedProcess([], 0, stdout="", stderr="") + bad = subprocess.CompletedProcess([], 1, stdout="", stderr="ERROR: cannot drop") + result, _, _ = self._run([self.REF, {}], psql=[ok, bad]) + self.assertFalse(result.passed) + self.assertTrue(any("could not drop the throwaway schema" in e + and "cannot drop" in e for e in result.errors), result.errors) + + def test_a_failed_create_is_reported_and_no_schema_query_is_run_on_it(self): + bad = subprocess.CompletedProcess([], 1, stdout="", stderr="ERROR: permission denied") + result, _, sig = self._run([self.REF], psql=[bad]) + self.assertFalse(result.passed) + self.assertTrue(any("could not create the throwaway schema" in e + for e in result.errors), result.errors) + self.assertEqual(sig.call_count, 1, "only the reference was queried") + + def test_an_unknown_reference_env_is_reported(self): + result, run, _ = self._run([], expected={"reference_env": "nope", "expected": {}}) + self.assertFalse(result.passed) + self.assertTrue(any("not one of the environments" in e for e in result.errors)) + run.assert_not_called() + + def test_the_scenario_is_dispatched_by_name(self): + scenario = Path("evals/datasets/tier_e/02_empty_schema_is_detected") + with patch.object(runner, "_load_expected", return_value={"expected": {}}), \ + patch.object(runner, "_can_connect_pg", return_value=True), \ + patch.object(runner, "_run_empty_schema_negative_control") as control: + runner.run_tier_e_scenario(scenario) + control.assert_called_once() + + def test_an_unknown_tier_e_scenario_is_still_an_error(self): + with patch.object(runner, "_load_expected", return_value={"expected": {}}), \ + patch.object(runner, "_can_connect_pg", return_value=True): + result = runner.run_tier_e_scenario(Path("evals/datasets/tier_e/99_unknown")) + self.assertIn("Unknown tier-E scenario: 99_unknown", result.errors) + + def test_the_scenario_files_exist_and_agree(self): + root = Path(__file__).resolve().parent.parent / "evals" + self.assertTrue((root / "datasets/tier_e/02_empty_schema_is_detected/NOTES.txt").is_file()) + expected = runner._load_expected("e", "02_empty_schema_is_detected") + self.assertEqual(expected["scenario"], "02_empty_schema_is_detected") + self.assertEqual(expected["expected"]["empty_schema_tables"], 0) + self.assertIs(expected["expected"]["diff_reports_missing_tables"], True) + + @pytest.mark.unit class TierXSqliteStructure(unittest.TestCase): """O-08: SQLite DDL execution and structural readback."""