diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 9a8cf79..821ae9a 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -158,7 +158,11 @@ jobs: # Prints a final block accounting for every test: PASSED / FAILED / # ERROR / SKIPPED (with reasons) / NOT RUN (deselected). --strict fails # the build on any skip. + # The API reads PGDATABASE (api/config.py). The schema lives in + # te_mgmt_dev, not the job-level default "postgres" (issue #45). - name: Full test suite — final result with skip accounting + env: + PGDATABASE: te_mgmt_dev run: python3 scripts/test_report.py --strict - name: Upload eval reports diff --git a/evals/runner.py b/evals/runner.py index c191a49..5a9c0a0 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -596,8 +596,9 @@ def _column_signature(db: str, schema: str) -> Dict[str, List[str]]: different namespaces compare equal when their structure matches. """ # The schema name is bound by psql via --set + :'sch' rather than pasted into - # the SQL text, so there is no string interpolation to reason about even - # though the value comes from the ENV_TARGETS constant. + # the SQL text. psql only interpolates variables in SQL it reads from a file + # or stdin, never in -c, so the query goes on stdin (issue #45: under -c it + # was a syntax error that this function used to report as "0 tables"). query = ( "SELECT table_name, column_name, data_type, is_nullable " "FROM information_schema.columns " @@ -605,11 +606,12 @@ def _column_signature(db: str, schema: str) -> Dict[str, List[str]]: "ORDER BY table_name, ordinal_position" ) r = subprocess.run( - ["psql", "-tA", "-d", db, "--set=sch=" + schema, "-c", query], - env=_pg_env(), capture_output=True, text=True, timeout=30, + ["psql", "-tA", "-v", "ON_ERROR_STOP=1", "-d", db, "--set=sch=" + schema], + input=query, env=_pg_env(), capture_output=True, text=True, timeout=30, ) if r.returncode != 0: - return {} + raise RuntimeError("schema query on " + db + "." + schema + " failed: " + + r.stderr.strip()[-300:]) sig: Dict[str, List[str]] = {} for line in r.stdout.splitlines(): parts = line.split("|") @@ -689,8 +691,13 @@ def _run_envs_have_identical_structure( deploys = [_deploy_env(env_name) for env_name, _, _ in targets] signatures: Dict[str, Dict[str, List[str]]] = {} + query_errors: List[str] = [] for env_name, db, schema in targets: - signatures[env_name] = _column_signature(db, schema) + try: + signatures[env_name] = _column_signature(db, schema) + except RuntimeError as exc: + signatures[env_name] = {} + query_errors.append("env " + env_name + ": " + str(exc)) table_counts = {e: len(sig) for e, sig in signatures.items()} result.actual = { @@ -698,7 +705,8 @@ def _run_envs_have_identical_structure( "table_counts": table_counts, } - errors: List[str] = [] + # A failed query is not an empty schema; report it as what it is. + errors: List[str] = list(query_errors) if exp.get("all_envs_deploy_exit_zero") is True: for d in deploys: diff --git a/tests/run_all_tests.sql b/tests/run_all_tests.sql index 8dd4034..abec3d9 100644 --- a/tests/run_all_tests.sql +++ b/tests/run_all_tests.sql @@ -32,11 +32,17 @@ -- ── Step 1: Load the test framework (results table + assertion functions) ───── --- Set search_path so framework objects and assertions resolve without schema prefix. +-- The harness keeps its own objects (test_run_results, assert_*, report_*) in +-- te_test_harness so they never land in the schema under test, where Tier E +-- would see them as drift (issue #45). It comes first on search_path, so the +-- framework's unqualified CREATEs go there; T&E tables still resolve unqualified +-- from the env schema, and the suites address it via te.schema_name anyway. +CREATE SCHEMA IF NOT EXISTS te_test_harness; + -- Expose per-env vars via current_setting() for PL/pgSQL DO blocks (:'var' doesn't -- substitute inside $dollar-quoted$ strings in psql 18). SELECT - set_config('search_path', :'schema_name' || ',public', false), + set_config('search_path', 'te_test_harness,' || :'schema_name' || ',public', false), set_config('te.schema_name', :'schema_name', false), set_config('te.app_user', :'app_user', false), set_config('te.conn_limit', :'conn_limit', false); @@ -70,7 +76,7 @@ SELECT skipped, pass_rate, suite_status -FROM :"schema_name".report_suite_summary(); +FROM te_test_harness.report_suite_summary(); \echo '' \echo '============================================================' @@ -83,7 +89,7 @@ SELECT expected, actual, message -FROM :"schema_name".report_failures(); +FROM te_test_harness.report_failures(); \echo '' \echo '============================================================' @@ -97,7 +103,7 @@ SELECT expected, actual, message -FROM :"schema_name".report_detail(); +FROM te_test_harness.report_detail(); \echo '' \echo '============================================================' @@ -111,7 +117,7 @@ SELECT skipped, pass_rate, overall -FROM :"schema_name".report_totals(); +FROM te_test_harness.report_totals(); \echo '' \echo '============================================================' diff --git a/tests/test_evals_tier_e_x.py b/tests/test_evals_tier_e_x.py index 1088f84..51efa99 100644 --- a/tests/test_evals_tier_e_x.py +++ b/tests/test_evals_tier_e_x.py @@ -9,8 +9,10 @@ from __future__ import annotations +import subprocess import sys import unittest +from unittest.mock import patch from pathlib import Path import pytest @@ -83,6 +85,47 @@ def test_env_targets_cover_all_four_environments(self): self.assertEqual(len({e[2] for e in runner.ENV_TARGETS}), 4) +@pytest.mark.unit +class TierEColumnSignatureQuery(unittest.TestCase): + """Issue #45: the schema query itself, not just the comparison. + + psql never interpolates :'var' inside -c, so the query must go on stdin, and + a failed query must not be reported as an empty schema. + """ + + def _scenario(self, completed): + expected = {"expected": {"min_tables_per_env": 12}, + "envs_under_test": ["dev"]} + result = runner.ScenarioResult(tier="e", name="t") + with patch.object(runner, "_deploy_env", + return_value={"env": "dev", "exit_code": 0}), \ + patch.object(runner.subprocess, "run", return_value=completed) as run: + runner._run_envs_have_identical_structure(result, expected) + return result, run + + def test_query_is_sent_on_stdin_not_via_dash_c(self): + ok = subprocess.CompletedProcess([], 0, stdout="t|c|text|NO\n", stderr="") + _, run = self._scenario(ok) + args, kwargs = run.call_args + self.assertNotIn("-c", args[0]) + self.assertIn(":'sch'", kwargs["input"]) + + def test_query_failure_is_reported_as_a_query_error(self): + bad = subprocess.CompletedProcess([], 1, stdout="", + stderr='ERROR: syntax error at or near ":"') + result, _ = self._scenario(bad) + self.assertFalse(result.passed) + self.assertTrue(any("schema query" in e and "syntax error" in e + for e in result.errors), result.errors) + + def test_genuinely_empty_schema_still_fails(self): + empty = subprocess.CompletedProcess([], 0, stdout="", stderr="") + result, _ = self._scenario(empty) + self.assertFalse(result.passed) + self.assertIn("env dev has 0 tables in its schema, expected >= 12 " + "(deploy may have partially failed)", result.errors) + + @pytest.mark.unit class TierXSqliteStructure(unittest.TestCase): """O-08: SQLite DDL execution and structural readback."""