diff --git a/README.md b/README.md index bc84491..0b84f54 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ bash tests/run_tests.sh dev ### Eval suite ```powershell -python evals\runner.py --tiers p,i,s,e # 26 scenarios: Tier P 23, I 1, S 1, E 1 +python evals\runner.py --tiers p,i,s,e # 27 scenarios: Tier P 23, I 1, S 1, E 2 ``` ### CI diff --git a/docs/TEST_CASES.md b/docs/TEST_CASES.md index 1f9fea9..314f585 100644 --- a/docs/TEST_CASES.md +++ b/docs/TEST_CASES.md @@ -1,7 +1,7 @@ # Test Catalogue > Auto-generated by `scripts/test_catalogue.py` — do not edit by hand. -> 779 tests across 40 files. +> 809 tests across 40 files. To regenerate: `python3 scripts/test_catalogue.py` @@ -1531,6 +1531,36 @@ To regenerate: `python3 scripts/test_catalogue.py` | `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | +| `TierEEmptySchemaControl.test_passes_when_an_empty_schema_reads_empty_and_the_comparison_flags_it` | unit | — | + +| `TierEEmptySchemaControl.test_fails_when_the_comparison_no_longer_reports_the_missing_tables` | unit | — | + +| `TierEEmptySchemaControl.test_fails_when_an_empty_schema_reports_tables` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_query_on_the_empty_schema_is_not_read_as_no_tables` | unit | — | + +| `TierEEmptySchemaControl.test_refuses_to_run_against_an_empty_reference_and_creates_nothing` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_reference_query_is_reported_and_creates_nothing` | unit | — | + +| `TierEEmptySchemaControl.test_creates_then_drops_the_throwaway_schema` | unit | — | + +| `TierEEmptySchemaControl.test_drops_the_throwaway_schema_even_when_the_check_fails` | unit | — | + +| `TierEEmptySchemaControl.test_never_drops_with_cascade` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_drop_is_reported` | unit | — | + +| `TierEEmptySchemaControl.test_a_failed_create_is_reported_and_no_schema_query_is_run_on_it` | unit | — | + +| `TierEEmptySchemaControl.test_an_unknown_reference_env_is_reported` | unit | — | + +| `TierEEmptySchemaControl.test_the_scenario_is_dispatched_by_name` | unit | — | + +| `TierEEmptySchemaControl.test_an_unknown_tier_e_scenario_is_still_an_error` | unit | — | + +| `TierEEmptySchemaControl.test_the_scenario_files_exist_and_agree` | unit | — | + | `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | | `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | @@ -1575,6 +1605,36 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_genuinely_empty_schema_still_fails` | — | — | +| `test_passes_when_an_empty_schema_reads_empty_and_the_comparison_flags_it` | — | — | + +| `test_fails_when_the_comparison_no_longer_reports_the_missing_tables` | — | — | + +| `test_fails_when_an_empty_schema_reports_tables` | — | — | + +| `test_a_failed_query_on_the_empty_schema_is_not_read_as_no_tables` | — | — | + +| `test_refuses_to_run_against_an_empty_reference_and_creates_nothing` | — | — | + +| `test_a_failed_reference_query_is_reported_and_creates_nothing` | — | — | + +| `test_creates_then_drops_the_throwaway_schema` | — | — | + +| `test_drops_the_throwaway_schema_even_when_the_check_fails` | — | — | + +| `test_never_drops_with_cascade` | — | — | + +| `test_a_failed_drop_is_reported` | — | — | + +| `test_a_failed_create_is_reported_and_no_schema_query_is_run_on_it` | — | — | + +| `test_an_unknown_reference_env_is_reported` | — | — | + +| `test_the_scenario_is_dispatched_by_name` | — | — | + +| `test_an_unknown_tier_e_scenario_is_still_an_error` | — | — | + +| `test_the_scenario_files_exist_and_agree` | — | — | + | `test_placeholders_are_substituted` | — | — | | `test_every_known_table_has_a_substitution` | — | — | diff --git a/evals/FAILURE_MODES.md b/evals/FAILURE_MODES.md index c68bd05..004c6eb 100644 --- a/evals/FAILURE_MODES.md +++ b/evals/FAILURE_MODES.md @@ -79,6 +79,9 @@ Tier S initial scope: only S1. - ~~**Cross-environment structural equivalence** (Dev vs Test vs Staging vs Prod) — Tier E, future.~~ *Now covered as of 2026-09-10:* Tier E scenario 01 compares all four deployed schemas column by column and treats reordered columns as drift. Closes VCRM BR-01. + Scenario 02 is its negative control: against the real database, an empty schema must + read as empty and comparing it with Dev must report the missing tables, so 01 cannot + pass a deploy that wrote nothing. - **Domain-rule deep dives beyond suite 05** — Tier D, future. - **Validator behaviour on >128KB single field** — beyond the current 50KB eval and Python `csv` default field-size assumptions. diff --git a/evals/HANDOFF.md b/evals/HANDOFF.md index 0f2f5f6..59d6924 100644 --- a/evals/HANDOFF.md +++ b/evals/HANDOFF.md @@ -6,7 +6,9 @@ > current state. Affected lines are marked inline. > > - **Tier E** — implemented. `evals/datasets/tier_e/01_envs_have_identical_structure`. -> Runs in the `integration-postgres` CI job. Closes VCRM BR-01. +> Runs in the `integration-postgres` CI job. Closes VCRM BR-01. Scenario +> `02_empty_schema_is_detected` is its negative control: it shows 01 can see a deploy +> that wrote nothing. > - **Tier X** — implemented. `evals/datasets/tier_x/01_sqlite_schema_structure`. > Deliberately **not** in CI per [ADR-0001](../docs/adr/0001-postgresql-first.md) > (PostgreSQL first); run it with `make evals-engines`. diff --git a/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt b/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt new file mode 100644 index 0000000..c4300a1 --- /dev/null +++ b/evals/datasets/tier_e/02_empty_schema_is_detected/NOTES.txt @@ -0,0 +1,42 @@ +Tier E scenario 02 - negative control for scenario 01 +===================================================== + +What it proves +-------------- +Scenario 01 passes when Test, Staging and Prod have the same structure as Dev. +That is only worth anything if "a deploy wrote nothing" is something it can see. +This scenario shows that, against the real database: + +1. A schema with no tables is read as having no tables by the same query + scenario 01 uses (_column_signature), and a failed query is reported as a + failure rather than as "no tables" (the confusion behind issue #45). +2. Comparing Dev's real schema with that empty one reports the missing tables + (_diff_signatures), so scenario 01 would fail on it. + +If either stops being true, scenario 01 could go green while a deploy silently +wrote nothing. + +How +--- +It creates a throwaway empty schema "_te_empty_neg_ctrl" in Dev's database, +reads its signature, compares it with Dev's, and drops it again. The drop never +uses CASCADE: if anything else is found in that schema the scenario fails rather +than deleting what is not its own. + +It refuses to run (fails) if Dev has fewer than min_reference_tables tables, +because comparing nothing with nothing would prove nothing. Scenario 01 deploys +Dev first, so run them together (the default for --tiers e); with --only, deploy +Dev beforehand. + +Relationship to the unit tests +------------------------------ +tests/test_evals_tier_e_x.py covers the same logic with a mocked psql. This is the +same claim against a real server, which is what mocks cannot show. + +Fails rather than skips +----------------------- +No PostgreSQL means a failure, as for scenario 01 (FIXES_APPLIED.md F5). + +Run +--- + python3 evals/runner.py --tiers e --verbose diff --git a/evals/expected/tier_e/02_empty_schema_is_detected.json b/evals/expected/tier_e/02_empty_schema_is_detected.json new file mode 100644 index 0000000..54647df --- /dev/null +++ b/evals/expected/tier_e/02_empty_schema_is_detected.json @@ -0,0 +1,10 @@ +{ + "scenario": "02_empty_schema_is_detected", + "description": "Negative control for scenario 01: a schema with no tables must read as empty, and comparing it with Dev's real schema must report the missing tables, so the parity eval cannot pass a deploy that wrote nothing.", + "reference_env": "dev", + "expected": { + "min_reference_tables": 12, + "empty_schema_tables": 0, + "diff_reports_missing_tables": true + } +} diff --git a/evals/runner.py b/evals/runner.py index ffffc25..4dc5627 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -677,6 +677,9 @@ def run_tier_e_scenario(scenario_dir: Path) -> ScenarioResult: if name == "01_envs_have_identical_structure": return _run_envs_have_identical_structure(result, expected) + if name == "02_empty_schema_is_detected": + return _run_empty_schema_negative_control(result, expected) + result.errors.append("Unknown tier-E scenario: " + name) return result @@ -748,6 +751,112 @@ def _run_envs_have_identical_structure( return result +# A schema that only the negative control below creates and drops. It is never +# dropped with CASCADE: if something else is found in it, the control fails +# rather than deleting what is not its own. +_EMPTY_CONTROL_SCHEMA = "_te_empty_neg_ctrl" + + +def _psql_run(db: str, sql: str) -> "subprocess.CompletedProcess[str]": + """Run one SQL statement on `db`; the caller reads returncode and stderr.""" + return subprocess.run( + ["psql", "-d", db, "-v", "ON_ERROR_STOP=1", "-c", sql], + env=_pg_env(), capture_output=True, text=True, timeout=30, + ) + + +def _run_empty_schema_negative_control( + result: ScenarioResult, expected: Dict[str, Any] +) -> ScenarioResult: + """Negative control for scenario 01: show that it can see "nothing deployed". + + Scenario 01 passes when every environment has the same structure as dev. That + only means something if a schema a deploy left empty is read as empty and the + comparison then reports it. This creates a throwaway empty schema beside dev's + and checks both, against the real database: + + 1. the real schema query returns no tables for it (and a failed query is a + failure, not an empty schema: the confusion behind issue #45); + 2. the real comparison between dev's schema and the empty one reports the + missing tables. + + If either stops being true, scenario 01 could pass while a deploy wrote + nothing. The pure logic is unit-tested with a mocked psql; this is the same + claim made against a real server. + """ + exp = expected.get("expected", {}) + ref_env = expected.get("reference_env", "dev") + target = next((t for t in ENV_TARGETS if t[0] == ref_env), None) + if target is None: + result.errors.append("reference_env " + ref_env + " is not one of the " + "environments in ENV_TARGETS") + return result + _, db, ref_schema = target + + # Without a populated reference the comparison below is "nothing against + # nothing" and would prove nothing, so that is an error, not a pass. + try: + ref = _column_signature(db, ref_schema) + except RuntimeError as exc: + result.errors.append("reference env " + ref_env + ": " + str(exc)) + return result + min_reference = int(exp.get("min_reference_tables", 12)) + if len(ref) < min_reference: + result.errors.append( + "reference env " + ref_env + " has " + str(len(ref)) + " tables, " + "expected >= " + str(min_reference) + ". Deploy it first (scenario 01 " + "does); an empty reference would make this control compare nothing " + "with nothing." + ) + return result + + quoted = '"' + _EMPTY_CONTROL_SCHEMA + '"' + made = _psql_run(db, "DROP SCHEMA IF EXISTS " + quoted + + "; CREATE SCHEMA " + quoted) + if made.returncode != 0: + result.errors.append("could not create the throwaway schema " + + _EMPTY_CONTROL_SCHEMA + " in " + db + ": " + + made.stderr.strip()[-300:]) + return result + + errors: List[str] = [] + try: + try: + empty = _column_signature(db, _EMPTY_CONTROL_SCHEMA) + except RuntimeError as exc: + empty = None + errors.append("the schema query failed on an empty schema, which " + "must not be reported as 'no tables': " + str(exc)) + if empty is not None: + want = int(exp.get("empty_schema_tables", 0)) + diffs = _diff_signatures(ref_env, ref, "empty", empty) + result.actual = { + "reference_tables": len(ref), + "empty_schema_tables": len(empty), + "differences_reported": len(diffs), + } + if len(empty) != want: + errors.append("an empty schema returned " + str(len(empty)) + + " tables, expected " + str(want)) + if exp.get("diff_reports_missing_tables") is True and not any( + " is missing tables present in " in d for d in diffs): + errors.append( + "comparing " + ref_env + " (" + str(len(ref)) + " tables) with " + "an empty schema reported no missing tables, so scenario 01 " + "would pass a deploy that wrote nothing" + ) + finally: + dropped = _psql_run(db, "DROP SCHEMA IF EXISTS " + quoted) + if dropped.returncode != 0: + errors.append("could not drop the throwaway schema " + + _EMPTY_CONTROL_SCHEMA + " (it is left in " + db + + "): " + dropped.stderr.strip()[-300:]) + + result.errors = errors + result.passed = not errors + return result + + # --------------------------------------------------------------------------- # Tier X — Cross-engine structural coverage (VCRM BR-02) # diff --git a/tests/unit/test_evals_tier_e_x.py b/tests/unit/test_evals_tier_e_x.py index f63d45a..de1e908 100644 --- a/tests/unit/test_evals_tier_e_x.py +++ b/tests/unit/test_evals_tier_e_x.py @@ -126,6 +126,139 @@ def test_genuinely_empty_schema_still_fails(self): "(deploy may have partially failed)", result.errors) +@pytest.mark.unit +class TierEEmptySchemaControl(unittest.TestCase): + """Scenario 02, the negative control for scenario 01. + + Against a real server it proves scenario 01 can see "a deploy wrote nothing". + Here psql and the schema query are mocked, to pin the control's own logic: + above all that it FAILS when the comparison stops reporting an empty schema, + since a control that cannot fail is worth nothing. + """ + + REF = {"t%d" % i: ["id:uuid:NO"] for i in range(12)} + EXPECTED = { + "reference_env": "dev", + "expected": {"min_reference_tables": 12, "empty_schema_tables": 0, + "diff_reports_missing_tables": True}, + } + + def _run(self, signatures, psql=None, diff=None, expected=None): + """Run the control; returns (result, psql mock, signature mock).""" + ok = subprocess.CompletedProcess([], 0, stdout="", stderr="") + result = runner.ScenarioResult(tier="e", name="02_empty_schema_is_detected") + with patch.object(runner, "_column_signature", side_effect=signatures) as sig, \ + patch.object(runner, "_psql_run", + **({"side_effect": psql} if psql else {"return_value": ok})) as run: + if diff is None: + runner._run_empty_schema_negative_control(result, expected or self.EXPECTED) + else: + with patch.object(runner, "_diff_signatures", return_value=diff): + runner._run_empty_schema_negative_control(result, expected or self.EXPECTED) + return result, run, sig + + @staticmethod + def _sql(run): + return [c.args[1] for c in run.call_args_list] + + def test_passes_when_an_empty_schema_reads_empty_and_the_comparison_flags_it(self): + result, run, _ = self._run([self.REF, {}]) + self.assertTrue(result.passed, result.errors) + self.assertEqual(result.actual["reference_tables"], 12) + self.assertEqual(result.actual["empty_schema_tables"], 0) + self.assertGreaterEqual(result.actual["differences_reported"], 1) + + def test_fails_when_the_comparison_no_longer_reports_the_missing_tables(self): + # The point of a control: neutralise the thing it guards and it must go red. + result, _, _ = self._run([self.REF, {}], diff=[]) + self.assertFalse(result.passed) + self.assertTrue(any("would pass a deploy that wrote nothing" in e + for e in result.errors), result.errors) + + def test_fails_when_an_empty_schema_reports_tables(self): + result, _, _ = self._run([self.REF, {"x": ["id:uuid:NO"]}]) + self.assertFalse(result.passed) + self.assertIn("an empty schema returned 1 tables, expected 0", result.errors) + + def test_a_failed_query_on_the_empty_schema_is_not_read_as_no_tables(self): + result, _, _ = self._run([self.REF, RuntimeError("connection lost")]) + self.assertFalse(result.passed) + self.assertTrue(any("must not be reported as 'no tables'" in e + and "connection lost" in e for e in result.errors), result.errors) + + def test_refuses_to_run_against_an_empty_reference_and_creates_nothing(self): + result, run, _ = self._run([{}]) + self.assertFalse(result.passed) + self.assertTrue(any("Deploy it first" in e for e in result.errors), result.errors) + run.assert_not_called() + + def test_a_failed_reference_query_is_reported_and_creates_nothing(self): + result, run, _ = self._run([RuntimeError("db down")]) + self.assertFalse(result.passed) + self.assertIn("reference env dev: db down", result.errors) + run.assert_not_called() + + def test_creates_then_drops_the_throwaway_schema(self): + _, run, _ = self._run([self.REF, {}]) + sql = self._sql(run) + self.assertEqual(len(sql), 2) + self.assertIn("CREATE SCHEMA", sql[0]) + self.assertTrue(sql[1].startswith("DROP SCHEMA IF EXISTS")) + self.assertIn(runner._EMPTY_CONTROL_SCHEMA, sql[1]) + + def test_drops_the_throwaway_schema_even_when_the_check_fails(self): + _, run, _ = self._run([self.REF, RuntimeError("boom")]) + self.assertTrue(self._sql(run)[-1].startswith("DROP SCHEMA IF EXISTS")) + + def test_never_drops_with_cascade(self): + _, run, _ = self._run([self.REF, {}]) + self.assertFalse(any("CASCADE" in q.upper() for q in self._sql(run)), self._sql(run)) + + def test_a_failed_drop_is_reported(self): + ok = subprocess.CompletedProcess([], 0, stdout="", stderr="") + bad = subprocess.CompletedProcess([], 1, stdout="", stderr="ERROR: cannot drop") + result, _, _ = self._run([self.REF, {}], psql=[ok, bad]) + self.assertFalse(result.passed) + self.assertTrue(any("could not drop the throwaway schema" in e + and "cannot drop" in e for e in result.errors), result.errors) + + def test_a_failed_create_is_reported_and_no_schema_query_is_run_on_it(self): + bad = subprocess.CompletedProcess([], 1, stdout="", stderr="ERROR: permission denied") + result, _, sig = self._run([self.REF], psql=[bad]) + self.assertFalse(result.passed) + self.assertTrue(any("could not create the throwaway schema" in e + for e in result.errors), result.errors) + self.assertEqual(sig.call_count, 1, "only the reference was queried") + + def test_an_unknown_reference_env_is_reported(self): + result, run, _ = self._run([], expected={"reference_env": "nope", "expected": {}}) + self.assertFalse(result.passed) + self.assertTrue(any("not one of the environments" in e for e in result.errors)) + run.assert_not_called() + + def test_the_scenario_is_dispatched_by_name(self): + scenario = Path("evals/datasets/tier_e/02_empty_schema_is_detected") + with patch.object(runner, "_load_expected", return_value={"expected": {}}), \ + patch.object(runner, "_can_connect_pg", return_value=True), \ + patch.object(runner, "_run_empty_schema_negative_control") as control: + runner.run_tier_e_scenario(scenario) + control.assert_called_once() + + def test_an_unknown_tier_e_scenario_is_still_an_error(self): + with patch.object(runner, "_load_expected", return_value={"expected": {}}), \ + patch.object(runner, "_can_connect_pg", return_value=True): + result = runner.run_tier_e_scenario(Path("evals/datasets/tier_e/99_unknown")) + self.assertIn("Unknown tier-E scenario: 99_unknown", result.errors) + + def test_the_scenario_files_exist_and_agree(self): + root = Path(__file__).resolve().parents[2] / "evals" + self.assertTrue((root / "datasets/tier_e/02_empty_schema_is_detected/NOTES.txt").is_file()) + expected = runner._load_expected("e", "02_empty_schema_is_detected") + self.assertEqual(expected["scenario"], "02_empty_schema_is_detected") + self.assertEqual(expected["expected"]["empty_schema_tables"], 0) + self.assertIs(expected["expected"]["diff_reports_missing_tables"], True) + + @pytest.mark.unit class TierXSqliteStructure(unittest.TestCase): """O-08: SQLite DDL execution and structural readback."""