diff --git a/Makefile b/Makefile index f256b58..ca4075b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ .PHONY: test-free test-gate test-evals test-e2e test-all \ + test-unit test-system test-sit test-performance test-accessibility test-uat \ lint lint-diff health \ eval-list eval-compare eval-summary select-tests \ csv-load csv-list csv-demo @@ -32,6 +33,35 @@ test-all: pytest tests/ --cov=build/csv --cov=evals --cov-report=term-missing python3 evals/runner.py --tiers p,i,s,e --verbose +# ── One category at a time (#167) ───────────────────────────────────────────── +# tests/ is split by what a test needs: unit (nothing), system (the API and +# PostgreSQL), system_integration (whole pipeline, all environments), then +# performance, accessibility and uat. See tests/README.md. + +# pytest exits 5 when a folder holds no tests yet; that is not a failure for a +# category that has none so far. +define run_category + python3 -m pytest $(1) -v; rc=$$?; if [ $$rc -eq 5 ]; then echo "No tests in $(1) yet."; exit 0; fi; exit $$rc +endef + +test-unit: + $(call run_category,tests/unit) + +test-system: + $(call run_category,tests/system) + +test-sit: + $(call run_category,tests/system_integration) + +test-performance: + $(call run_category,tests/performance) + +test-accessibility: + $(call run_category,tests/accessibility) + +test-uat: + $(call run_category,tests/uat) + # ── Code quality ────────────────────────────────────────────────────────────── # Full flake8 + bandit scan over all Python source. diff --git a/README.md b/README.md index 8cf1d6a..bc84491 100644 --- a/README.md +++ b/README.md @@ -244,8 +244,8 @@ Interactive docs: `http://localhost:8000/docs` python -m pytest tests\ -m "unit or regression or security or snapshot" -q # Integration tests — requires running API + provisioned database -python -m pytest tests\test_api_integration.py -v -python -m pytest tests\test_te_loader_integration.py -v +python -m pytest tests\system\test_api_integration.py -v +python -m pytest tests\system\test_te_loader_integration.py -v # Per-environment contract tests python -m pytest tests\environments\ -v @@ -338,7 +338,7 @@ organisations ──< personnel | Connection Limit | 10 | 15 | 25 | 50 | | Seed Data | ✅ | ✅ | ❌ | ❌ | -Staging and Prod are **never seeded**. The `tests/environments/` contract tests +Staging and Prod are **never seeded**. The `tests/system_integration/environments/` contract tests verify this automatically. --- diff --git a/api/auth.py b/api/auth.py index 93c920a..8294df3 100644 --- a/api/auth.py +++ b/api/auth.py @@ -78,7 +78,7 @@ async def tenant_scope(authorization: str = Header(default=""), # M2c (#94): which roles may perform each action. docs/ROLES.md shows the same -# table, and tests/test_roles.py checks that the two agree. +# table, and tests/system/test_roles.py checks that the two agree. PERMISSIONS: dict[str, frozenset[str]] = { "preview": frozenset({"uploader", "tenant_admin"}), "upload": frozenset({"uploader", "tenant_admin"}), diff --git a/api/routers/audit_routes.py b/api/routers/audit_routes.py index 49e7d6c..d9e33c3 100644 --- a/api/routers/audit_routes.py +++ b/api/routers/audit_routes.py @@ -5,7 +5,7 @@ O-04 update: the table is no longer delete-only. Successful uploads also write a row, through the shared writer in api/services/audit.py, so this feed now covers creations as well as destructions. Integration coverage lives in -tests/test_api_integration.py::AuditLogTrail.""" +tests/system/test_api_integration.py::AuditLogTrail.""" from __future__ import annotations diff --git a/docs/API_INTEGRATION.md b/docs/API_INTEGRATION.md index 28dac12..801a119 100644 --- a/docs/API_INTEGRATION.md +++ b/docs/API_INTEGRATION.md @@ -237,8 +237,8 @@ Per the repository's no-skip policy, the integration group **fails** with remediation text when the database is unreachable rather than skipping. ```bash -python -m pytest tests/test_api_unit.py -m unit # no database -python -m pytest tests/test_api_unit.py tests/test_api_integration.py # both files +python -m pytest tests/unit/test_api_unit.py -m unit # no database +python -m pytest tests/unit/test_api_unit.py tests/system/test_api_integration.py # both files python scripts/test_report.py --strict # whole suite ``` @@ -254,8 +254,8 @@ with remediation text rather than skipping, per the no-skip policy. ## Also required -CI installs only `requirements-dev.txt`. Because `tests/test_api_unit.py` and -`tests/test_api_integration.py` import FastAPI at module level, both +CI installs only `requirements-dev.txt`. Because `tests/unit/test_api_unit.py` and +`tests/system/test_api_integration.py` import FastAPI at module level, both workflows need: ```yaml diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b9e0e48..35c5ccd 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -50,15 +50,15 @@ The three layers can break independently, so we keep them physically separate. T | `tests/run_tests.sh` | Bash wrapper that sources `config.local.env` | | `tests/run_python_tests.ps1` | Windows runner — invoked by the GitHub Actions workflow | | `tests/conftest.py` | pytest env-var isolation between tests | -| `tests/test_csv_validator.py` | unittest for `build/csv/validator.py` | -| `tests/test_csv_utilise.py` | unit tests for `build/csv_utilise.sh` argument parsing | -| `tests/test_csv_loader_arbitrary_shapes.py` | integration: arbitrary CSV shapes through loader → PG (skips without PG) | -| `tests/test_e2e_pipeline.py` | e2e: CSV → validate → load → verify (DB half skips without PG) | -| `tests/test_parity.py` | cross-environment row-count / schema parity (skips without PG) | -| `tests/test_regression.py` | pinned tests for previously found bug classes | -| `tests/test_security.py` | static credential/SQL-pattern scans | -| `tests/test_snapshot.py` | golden-file output comparisons (`tests/snapshots/`) | -| `tests/test_evals_runner.py` | unittest for `evals/runner.py` itself | +| `tests/unit/test_csv_validator.py` | unittest for `build/csv/validator.py` | +| `tests/unit/test_csv_utilise.py` | unit tests for `build/csv_utilise.sh` argument parsing | +| `tests/system_integration/test_csv_loader_arbitrary_shapes.py` | integration: arbitrary CSV shapes through loader → PG (skips without PG) | +| `tests/system_integration/test_e2e_pipeline.py` | e2e: CSV → validate → load → verify (DB half skips without PG) | +| `tests/system_integration/test_parity.py` | cross-environment row-count / schema parity (skips without PG) | +| `tests/unit/test_regression.py` | pinned tests for previously found bug classes | +| `tests/unit/test_security.py` | static credential/SQL-pattern scans | +| `tests/unit/test_snapshot.py` | golden-file output comparisons (`tests/snapshots/`) | +| `tests/unit/test_evals_runner.py` | unittest for `evals/runner.py` itself | ### `evals/` — data-driven scenarios @@ -110,7 +110,7 @@ These are left at the repo root and excluded from the layer model. They can move ## Path conventions inside the layers - Code inside `evals/runner.py` computes `PROJECT_ROOT = Path(__file__).resolve().parent.parent` and reaches into `build/csv/validator.py`, `build/environments/env_dev.sql`, `tests/run_all_tests.sql` from there. -- Code inside `tests/test_csv_validator.py` does `Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py"`. +- Code inside `tests/unit/test_csv_validator.py` does `Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py"`. - Shell scripts inside `build/` cd into their own directory and use relative paths (`./adapters/...`, `./schema/...`). - Nothing in `build/` references `tests/` or `evals/`. diff --git a/docs/ENGINE_PARITY_TODO.md b/docs/ENGINE_PARITY_TODO.md index 0febbb9..e848a9e 100644 --- a/docs/ENGINE_PARITY_TODO.md +++ b/docs/ENGINE_PARITY_TODO.md @@ -96,7 +96,7 @@ temp_documents -> references test_programs (independent of the chain above) Then, in the same commit, move each table from `known_missing_tables` into `expected_tables` in `evals/expected/tier_x/01_sqlite_schema_structure.json`. Tier X will fail until -you do — `tests/test_evals_tier_e_x.py` asserts that the two lists together +you do — `tests/unit/test_evals_tier_e_x.py` asserts that the two lists together account for all 12 tables, and that everything listed as missing is genuinely absent. That is the reminder working as designed. diff --git a/docs/ENV_VARIABLES.md b/docs/ENV_VARIABLES.md index da8868d..33df349 100644 --- a/docs/ENV_VARIABLES.md +++ b/docs/ENV_VARIABLES.md @@ -69,7 +69,7 @@ the environment SQL templates. Copy `build/config.env.example` to > **Staging and prod must never be seeded.** The `SEED_STAGING` and `SEED_PROD` > variables default to `false`. Changing them to `true` will insert test data -> into production. The `tests/test_parity.py` contract tests verify this. +> into production. The `tests/system_integration/test_parity.py` contract tests verify this. ### Additional build-script variables diff --git a/docs/GAP_ANALYSIS.md b/docs/GAP_ANALYSIS.md index d9e862e..b1b26ee 100644 --- a/docs/GAP_ANALYSIS.md +++ b/docs/GAP_ANALYSIS.md @@ -117,9 +117,9 @@ scenario still fails on drift in either direction. It also executes the DDL rather than `adapter_sqlite.sh`, so the adapter scripts themselves remain unexercised — a smaller, separate gap. -Also mitigated: `tests/test_parity.py::TestAllEnvironmentsHaveRequiredTables` +Also mitigated: `tests/system_integration/test_parity.py::TestAllEnvironmentsHaveRequiredTables` runs against all four PostgreSQL environments, and -`tests/test_evals_tier_e_x.py` unit-tests both tiers' comparison logic with +`tests/unit/test_evals_tier_e_x.py` unit-tests both tiers' comparison logic with negative controls for added columns, changed types, changed nullability, reordered columns, and missing or extra tables. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index a882851..83fa732 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -198,10 +198,10 @@ Or use the convenience script from the repo root: python -m pytest tests\ -m "unit or regression or security or snapshot" -q # Integration tests — requires running API + provisioned database -python -m pytest tests\test_api_integration.py -v +python -m pytest tests\system\test_api_integration.py -v # T&E mode integration tests -python -m pytest tests\test_te_loader_integration.py -v +python -m pytest tests\system\test_te_loader_integration.py -v # Per-environment contract tests python -m pytest tests\environments\ -v diff --git a/docs/ROLES.md b/docs/ROLES.md index 69bded2..e75cf30 100644 --- a/docs/ROLES.md +++ b/docs/ROLES.md @@ -1,6 +1,6 @@ # Roles and permissions (BR-36, #94) -Every member of a tenant has one role (ADR-0003). This table is enforced by `PERMISSIONS` in `api/auth.py`. `tests/test_roles.py` fails if the two disagree, and it tries every role against every route. +Every member of a tenant has one role (ADR-0003). This table is enforced by `PERMISSIONS` in `api/auth.py`. `tests/system/test_roles.py` fails if the two disagree, and it tries every role against every route. | Action | Route | uploader | reviewer | tenant_admin | auditor | |---|---|---|---|---|---| diff --git a/docs/STARTER_GUIDE.md b/docs/STARTER_GUIDE.md index fa44d89..61abec6 100644 --- a/docs/STARTER_GUIDE.md +++ b/docs/STARTER_GUIDE.md @@ -57,7 +57,7 @@ Both modes go through the same `POST /api/csv/upload` endpoint, with a The same codebase runs against four isolated databases — Dev, Test, Staging, Prod — each with its own database name, schema, and app user (table in the main `README.md` under "Multi-Environment Setup"). Staging and Prod are never -seeded with sample data; `tests/environments/` enforces that automatically. +seeded with sample data; `tests/system_integration/environments/` enforces that automatically. If you're not sure which environment a command is touching, check `docs/DATABASE_ENVIRONMENTS.md` and `docs/ENV_VARIABLES.md` before running anything destructive. @@ -135,14 +135,14 @@ because of a specific past incident: - **All SQL identifiers (table/column names) go through `psycopg2.sql.Identifier()`** — never f-string interpolation. This is - AST-checked by `tests/test_security.py`. + AST-checked by `tests/unit/test_security.py`. - **pytest markers matter**: `unit`, `integration`, `e2e`, `regression`, `security`, `snapshot`, `parity`. Run `unit` (and friends) for a fast, DB-free sanity check; `integration` and `e2e` need a live, provisioned PostgreSQL. See `pytest.ini` for the full marker list. - **Staging and Prod are never seeded** — if you're touching provisioning scripts, don't make that possible even accidentally; there's a contract - test for it in `tests/environments/`. + test for it in `tests/system_integration/environments/`. ## If something doesn't work diff --git a/docs/TEST_CASES.md b/docs/TEST_CASES.md index 9dd0768..ef3e0b4 100644 --- a/docs/TEST_CASES.md +++ b/docs/TEST_CASES.md @@ -5,7 +5,7 @@ To regenerate: `python3 scripts/test_catalogue.py` -## `tests/test_accounts.py` +## `tests/system/test_accounts.py` | Test | Markers | Objective | |---|---|---| @@ -73,7 +73,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_failures_outside_the_window_start_a_new_count` | — | — | -## `tests/test_api_coverage.py` +## `tests/system/test_api_coverage.py` | Test | Markers | Objective | |---|---|---| @@ -233,107 +233,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_delete_blocked_returns_403` | — | — | -## `tests/test_api_hardening.py` - -| Test | Markers | Objective | -|---|---|---| -| `AuthFailsClosed.test_dev_may_run_without_a_key` | unit | — | - -| `AuthFailsClosed.test_any_key_is_accepted_in_dev` | unit | — | - -| `AuthFailsClosed.test_prod_without_a_key_refuses_to_start` | unit | — | - -| `AuthFailsClosed.test_staging_and_test_also_refuse` | unit | — | - -| `AuthFailsClosed.test_deployed_env_with_a_key_starts` | unit | — | - -| `RateLimiterAlgorithm.test_requests_under_budget_are_allowed` | unit | — | - -| `RateLimiterAlgorithm.test_request_over_budget_is_refused` | unit | — | - -| `RateLimiterAlgorithm.test_retry_after_points_at_the_oldest_hit_leaving_the_window` | unit | — | - -| `RateLimiterAlgorithm.test_window_expiry_frees_the_budget` | unit | — | - -| `RateLimiterAlgorithm.test_zero_budget_disables_the_limit` | unit | — | - -| `RateLimiterAlgorithm.test_read_and_write_scopes_are_independent` | unit | — | - -| `RateLimiterAlgorithm.test_clients_are_independent` | unit | — | - -| `RateLimiterAlgorithm.test_bucket_table_is_bounded` | unit | — | - -| `RateLimiterAlgorithm.test_reset_clears_state` | unit | — | - -| `RateLimitMiddleware.test_exceeding_the_write_budget_returns_429_with_retry_after` | unit | — | - -| `RateLimitMiddleware.test_health_is_never_rate_limited` | unit | — | - -| `RateLimitMiddleware.test_openapi_is_never_rate_limited` | unit | — | - -| `RateLimitMiddleware.test_disabled_budget_allows_everything` | unit | — | - -| `RateLimitMiddleware.test_forwarded_for_cannot_be_used_to_evade_the_limit` | unit | — | - -| `AuditWriterContract.test_only_known_actions_are_accepted` | unit | — | - -| `AuditWriterContract.test_unknown_action_raises_before_any_sql_runs` | unit | — | - -| `AuditWriterContract.test_valid_action_reaches_the_cursor_with_bound_parameters` | unit | — | - -| `AuditWriterContract.test_file_id_is_coerced_from_the_loaders_string_form` | unit | — | - -| `AuditWriterContract.test_unparseable_file_id_degrades_to_null_rather_than_raising` | unit | — | - -| `test_dev_may_run_without_a_key` | — | — | - -| `test_any_key_is_accepted_in_dev` | — | — | - -| `test_prod_without_a_key_refuses_to_start` | — | — | - -| `test_staging_and_test_also_refuse` | — | — | - -| `test_deployed_env_with_a_key_starts` | — | — | - -| `test_requests_under_budget_are_allowed` | — | — | - -| `test_request_over_budget_is_refused` | — | — | - -| `test_retry_after_points_at_the_oldest_hit_leaving_the_window` | — | — | - -| `test_window_expiry_frees_the_budget` | — | — | - -| `test_zero_budget_disables_the_limit` | — | — | - -| `test_read_and_write_scopes_are_independent` | — | — | - -| `test_clients_are_independent` | — | — | - -| `test_bucket_table_is_bounded` | — | — | - -| `test_reset_clears_state` | — | — | - -| `test_exceeding_the_write_budget_returns_429_with_retry_after` | — | — | - -| `test_health_is_never_rate_limited` | — | — | - -| `test_openapi_is_never_rate_limited` | — | — | - -| `test_disabled_budget_allows_everything` | — | — | - -| `test_forwarded_for_cannot_be_used_to_evade_the_limit` | — | — | - -| `test_only_known_actions_are_accepted` | — | — | - -| `test_unknown_action_raises_before_any_sql_runs` | — | — | - -| `test_valid_action_reaches_the_cursor_with_bound_parameters` | — | — | - -| `test_file_id_is_coerced_from_the_loaders_string_form` | — | — | - -| `test_unparseable_file_id_degrades_to_null_rather_than_raising` | — | — | - -## `tests/test_api_integration.py` +## `tests/system/test_api_integration.py` | Test | Markers | Objective | |---|---|---| @@ -445,59 +345,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_exhausted_pool_raises_within_timeout` | — | psycopg2's ThreadedConnectionPool.getconn() raises PoolError | -## `tests/test_api_unit.py` - -| Test | Markers | Objective | -|---|---|---| -| `UnitCsvParse.test_sanitize_columns_no_collision_on_stem_conflict` | unit | BUG-036 regression: ["foo","foo","foo_2"] must produce three | - -| `UnitCsvParse.test_sanitize_columns_dedupes_true_duplicates` | unit | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | - -| `UnitCsvParse.test_sanitize_columns_reserved_stems_get_suffix` | unit | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | - -| `RequestValidation.test_preview_rejects_empty_filename` | unit | — | - -| `RequestValidation.test_preview_rejects_empty_content` | unit | — | - -| `RequestValidation.test_preview_rejects_overlong_filename` | unit | — | - -| `RequestValidation.test_upload_rejects_unknown_mode` | unit | — | - -| `RequestValidation.test_upload_te_mode_requires_target_table` | unit | — | - -| `RequestValidation.test_oversized_upload_is_rejected` | unit | — | - -| `TableNameGuards.test_non_upload_tables_are_rejected` | security, unit | — | - -| `TableNameGuards.test_overlong_names_are_rejected` | security, unit | — | - -| `TableNameGuards.test_identifiers_are_never_string_formatted` | security, unit | Every SQL identifier must go through psycopg2.sql.Identifier. | - -| `test_sanitize_columns_no_collision_on_stem_conflict` | — | BUG-036 regression: ["foo","foo","foo_2"] must produce three | - -| `test_sanitize_columns_dedupes_true_duplicates` | — | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | - -| `test_sanitize_columns_reserved_stems_get_suffix` | — | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | - -| `test_preview_rejects_empty_filename` | — | — | - -| `test_preview_rejects_empty_content` | — | — | - -| `test_preview_rejects_overlong_filename` | — | — | - -| `test_upload_rejects_unknown_mode` | — | — | - -| `test_upload_te_mode_requires_target_table` | — | — | - -| `test_oversized_upload_is_rejected` | — | — | - -| `test_non_upload_tables_are_rejected` | — | — | - -| `test_overlong_names_are_rejected` | — | — | - -| `test_identifiers_are_never_string_formatted` | — | Every SQL identifier must go through psycopg2.sql.Identifier. | - -## `tests/test_audit_append_only.py` +## `tests/system/test_audit_append_only.py` | Test | Markers | Objective | |---|---|---| @@ -529,911 +377,1183 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_downgrade_removes_the_triggers_and_columns` | — | — | -## `tests/test_ci_quality_gate.py` +## `tests/system/test_failed_uploads.py` | Test | Markers | Objective | |---|---|---| -| `TestQualityGateStructure.test_has_free_tier_job` | — | — | +| `EveryRowFailed.test_every_row_failing_is_an_error_and_saves_nothing` | integration | — | -| `TestQualityGateStructure.test_has_integration_postgres_job` | — | — | +| `EveryRowFailed.test_failed_overwrite_keeps_the_previous_upload` | integration | — | -| `TestQualityGateStructure.test_triggers_on_pull_request` | — | — | +| `EveryRowFailed.test_partly_failed_upload_still_imports_and_reports_the_failures` | integration | — | -| `TestQualityGateStructure.test_triggers_on_push_to_main` | — | — | +| `test_every_row_failing_is_an_error_and_saves_nothing` | — | — | -| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_checkout` | — | — | +| `test_failed_overwrite_keeps_the_previous_upload` | — | — | -| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_python_setup` | — | — | +| `test_partly_failed_upload_still_imports_and_reports_the_failures` | — | — | -| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_artifact_upload` | — | — | +## `tests/system/test_header_mapping.py` -| `TestAllJobsHaveConsistentStructure.test_all_jobs_install_dev_deps` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `ApplyHeaderMapping.test_no_mapping_returns_headers_unchanged` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_row_exists` | — | — | +| `ApplyHeaderMapping.test_rename_replaces_only_the_named_header` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_is_accepted_or_deferred` | — | — | +| `ApplyHeaderMapping.test_blank_rename_keeps_the_original` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_section_exists` | — | — | +| `ApplyHeaderMapping.test_unknown_original_is_rejected` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_is_not_marked_closed` | — | — | +| `ApplyHeaderMapping.test_original_listed_twice_is_rejected` | unit | — | -| `test_has_free_tier_job` | — | — | +| `ApplyHeaderMapping.test_rename_colliding_with_an_untouched_column_is_rejected` | unit | — | -| `test_has_integration_postgres_job` | — | — | +| `ApplyHeaderMapping.test_collision_is_judged_after_sanitising` | unit | — | -| `test_triggers_on_pull_request` | — | — | +| `ApplyHeaderMapping.test_swapping_two_names_is_allowed` | unit | — | -| `test_triggers_on_push_to_main` | — | — | +| `HeaderMappingOverTheApi.test_renamed_header_becomes_the_database_column` | integration | — | -| `test_all_jobs_have_checkout` | — | — | +| `HeaderMappingOverTheApi.test_upload_without_a_mapping_keeps_csv_headers` | integration | — | -| `test_all_jobs_have_python_setup` | — | — | +| `HeaderMappingOverTheApi.test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | integration | — | -| `test_all_jobs_have_artifact_upload` | — | — | +| `HeaderMappingOverTheApi.test_rename_that_collides_with_another_column_is_an_error` | integration | — | -| `test_all_jobs_install_dev_deps` | — | — | +| `HeaderMappingOverTheApi.test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | integration | — | -| `test_g2_row_exists` | — | — | +| `test_no_mapping_returns_headers_unchanged` | — | — | -| `test_g2_is_accepted_or_deferred` | — | — | +| `test_rename_replaces_only_the_named_header` | — | — | -| `test_g2_section_exists` | — | — | +| `test_blank_rename_keeps_the_original` | — | — | -| `test_g2_is_not_marked_closed` | — | — | +| `test_unknown_original_is_rejected` | — | — | -## `tests/test_csv_loader_arbitrary_shapes.py` +| `test_original_listed_twice_is_rejected` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `CsvLoaderArbitraryShapes.test_shape_tiny` | — | — | +| `test_rename_colliding_with_an_untouched_column_is_rejected` | — | — | -| `CsvLoaderArbitraryShapes.test_shape_medium` | — | — | +| `test_collision_is_judged_after_sanitising` | — | — | -| `CsvLoaderArbitraryShapes.test_shape_skinny` | — | — | +| `test_swapping_two_names_is_allowed` | — | — | -| `test_shape_tiny` | — | — | +| `test_renamed_header_becomes_the_database_column` | — | — | -| `test_shape_medium` | — | — | +| `test_upload_without_a_mapping_keeps_csv_headers` | — | — | -| `test_shape_skinny` | — | — | +| `test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | — | — | + +| `test_rename_that_collides_with_another_column_is_an_error` | — | — | -## `tests/test_csv_utilise.py` +| `test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | — | — | + +## `tests/system/test_import_counts.py` | Test | Markers | Objective | |---|---|---| -| `CsvUtiliseArgumentParsing.test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | - -| `CsvUtiliseArgumentParsing.test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | +| `ImportCountsArePersisted.test_dynamic_import_persists_all_four_counts` | integration | — | -| `CsvUtiliseArgumentParsing.test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | +| `ImportCountsArePersisted.test_te_import_persists_counts` | integration | — | -| `CsvUtiliseArgumentParsing.test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | +| `test_dynamic_import_persists_all_four_counts` | — | — | -| `CsvUtiliseArgumentParsing.test_export_requires_two_positional_args` | — | `export` needs both and → exit 1 otherwise. | +| `test_te_import_persists_counts` | — | — | -| `CsvUtiliseArgumentParsing.test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | +## `tests/system/test_issue_04_multi_file_upload.py` -| `CsvUtiliseArgumentParsing.test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | +| Test | Markers | Objective | +|---|---|---| +| `MultipleFileUpload.test_many_files_upload_in_one_session` | integration | Uploading several files in sequence must all succeed. | -| `CsvUtiliseArgumentParsing.test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | +| `MultipleFileUpload.test_every_uploaded_file_is_listed` | integration | All uploads appear in /api/csv/files — this drives 'Migrated files'. | -| `CsvUtiliseArgumentParsing.test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | +| `MultipleFileUpload.test_each_file_is_independently_readable` | integration | Rows from one upload must not leak into another's table. | -| `test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | +| `MultipleFileUpload.test_narrow_and_wide_files_both_load` | integration | One column and many columns must both be accepted. | -| `test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | +| `MultipleFileUpload.test_files_with_different_schemas_coexist` | integration | Unrelated column sets must not collide — no shared fixed schema. | -| `test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | +| `MultipleFileUpload.test_a_larger_file_loads_completely` | integration | Row count is not silently truncated. | -| `test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | +| `test_many_files_upload_in_one_session` | — | Uploading several files in sequence must all succeed. | -| `test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | +| `test_every_uploaded_file_is_listed` | — | All uploads appear in /api/csv/files — this drives 'Migrated files'. | -| `test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | +| `test_each_file_is_independently_readable` | — | Rows from one upload must not leak into another's table. | -| `test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | +| `test_narrow_and_wide_files_both_load` | — | One column and many columns must both be accepted. | -| `test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | +| `test_files_with_different_schemas_coexist` | — | Unrelated column sets must not collide — no shared fixed schema. | -| `test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | +| `test_a_larger_file_loads_completely` | — | Row count is not silently truncated. | -## `tests/test_csv_validator.py` +## `tests/system/test_issue_05_import_summary.py` | Test | Markers | Objective | |---|---|---| -| `CsvValidatorTests.test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | +| `ImportSummaryCounts.test_all_four_counts_are_returned` | integration | — | -| `CsvValidatorTests.test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | +| `ImportSummaryCounts.test_total_rows_counts_data_rows_not_the_header` | integration | — | -| `CsvValidatorTests.test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | +| `ImportSummaryCounts.test_created_count_matches_rows_actually_inserted` | integration | — | -| `CsvValidatorTests.test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | +| `ImportSummaryCounts.test_duplicate_count_reflects_repeated_rows_in_the_file` | integration | Two identical rows plus one distinct: 1 duplicate skipped. | -| `test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | +| `ImportSummaryCounts.test_failed_count_reflects_rows_that_could_not_be_cast` | integration | With an int8 column, a non-numeric value must count as failed. | -| `test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | +| `ImportSummaryCounts.test_failed_rows_are_reported_with_detail` | integration | A count alone is not actionable — the offending row must be named. | -| `test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | +| `ImportSummaryCounts.test_counts_reconcile_for_a_clean_file` | integration | — | -| `test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | +| `ImportSummaryCounts.test_counts_reconcile_for_a_mixed_file` | integration | One good row, one in-file duplicate, one uncastable value. | -## `tests/test_e2e_pipeline.py` +| `ImportSummaryCounts.test_counts_are_per_batch_not_cumulative` | integration | A second, larger upload must report its own totals, not a running sum. | -| Test | Markers | Objective | -|---|---|---| -| `E2EPipelineValidateOnly.test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | +| `test_all_four_counts_are_returned` | — | — | -| `E2EPipelineValidateOnly.test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | +| `test_total_rows_counts_data_rows_not_the_header` | — | — | -| `E2EPipelineValidateOnly.test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | +| `test_created_count_matches_rows_actually_inserted` | — | — | -| `E2EPipelineValidateOnly.test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | +| `test_duplicate_count_reflects_repeated_rows_in_the_file` | — | Two identical rows plus one distinct: 1 duplicate skipped. | -| `E2EPipelineWithDatabase.test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | +| `test_failed_count_reflects_rows_that_could_not_be_cast` | — | With an int8 column, a non-numeric value must count as failed. | -| `E2EPipelineWithDatabase.test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | +| `test_failed_rows_are_reported_with_detail` | — | A count alone is not actionable — the offending row must be named. | -| `test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | +| `test_counts_reconcile_for_a_clean_file` | — | — | -| `test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | +| `test_counts_reconcile_for_a_mixed_file` | — | One good row, one in-file duplicate, one uncastable value. | -| `test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | +| `test_counts_are_per_batch_not_cumulative` | — | A second, larger upload must report its own totals, not a running sum. | -| `test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | +## `tests/system/test_migrations.py` -| `test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | +| Test | Markers | Objective | +|---|---|---| +| `MigrationsOnScratchSchema.test_upgrade_head_creates_tables_and_unique_constraints` | integration | — | -| `test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | +| `MigrationsOnScratchSchema.test_control_schema_is_created_and_removed` | integration | — | -## `tests/test_evals_runner.py` +| `MigrationsOnScratchSchema.test_downgrade_reverses_each_step` | integration | — | -| Test | Markers | Objective | -|---|---|---| -| `EvalsRunnerTests.test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | +| `MigrationsOnScratchSchema.test_baseline_adopts_a_fallback_created_schema` | integration | — | -| `EvalsRunnerTests.test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | +| `MigrationsOnScratchSchema.test_existing_duplicates_stop_the_upgrade_with_remediation` | integration | — | -| `EvalsRunnerTests.test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | +| `RegistryUniqueness.test_bootstrap_brings_the_schema_to_head` | integration | — | -| `EvalsRunnerTests.test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | +| `RegistryUniqueness.test_duplicate_hash_is_rejected_by_the_database` | integration | — | -| `EvalsRunnerTests.test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | +| `RegistryUniqueness.test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | integration | — | -| `EvalsRunnerTests.test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | +| `RegistryUniqueness.test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | integration | — | -| `EvalsRunnerTests.test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | +| `test_upgrade_head_creates_tables_and_unique_constraints` | — | — | -| `test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | +| `test_control_schema_is_created_and_removed` | — | — | -| `test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | +| `test_downgrade_reverses_each_step` | — | — | -| `test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | +| `test_baseline_adopts_a_fallback_created_schema` | — | — | -| `test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | +| `test_existing_duplicates_stop_the_upgrade_with_remediation` | — | — | -| `test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | +| `test_bootstrap_brings_the_schema_to_head` | — | — | -| `test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | +| `test_duplicate_hash_is_rejected_by_the_database` | — | — | -| `test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | +| `test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | — | — | + +| `test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | — | — | -## `tests/test_evals_tier_e_x.py` +## `tests/system/test_oidc.py` | Test | Markers | Objective | |---|---|---| -| `TierEDiffSignatures.test_identical_schemas_report_no_differences` | unit | — | +| `TokenVerification.test_valid_token_yields_the_username` | unit | — | -| `TierEDiffSignatures.test_namespace_is_not_part_of_the_comparison` | unit | — | +| `TokenVerification.test_rejected_tokens` | unit | — | -| `TierEDiffSignatures.test_missing_table_is_caught` | unit | — | +| `OidcSignIn.test_valid_token_for_a_member_gets_a_session` | integration | — | -| `TierEDiffSignatures.test_extra_table_is_caught` | unit | — | +| `OidcSignIn.test_valid_token_but_no_membership_or_account_is_401` | integration | — | -| `TierEDiffSignatures.test_added_column_is_caught` | unit | — | +| `OidcSignIn.test_invalid_token_is_401` | integration | — | -| `TierEDiffSignatures.test_changed_type_is_caught` | unit | — | +| `OidcSignIn.test_inactive_user_is_401` | integration | — | -| `TierEDiffSignatures.test_changed_nullability_is_caught` | unit | — | +| `OidcSignIn.test_route_is_404_when_oidc_is_not_configured` | integration | — | -| `TierEDiffSignatures.test_reordered_columns_are_caught_as_drift` | unit | — | +| `ConfigFailsClosed.test_off_by_default_and_complete_config_passes` | unit | — | -| `TierEDiffSignatures.test_env_targets_cover_all_four_environments` | unit | — | +| `ConfigFailsClosed.test_missing_audience_or_insecure_jwks_url_refuses_to_start` | unit | — | -| `TierEColumnSignatureQuery.test_query_is_sent_on_stdin_not_via_dash_c` | unit | — | +| `test_valid_token_yields_the_username` | — | — | -| `TierEColumnSignatureQuery.test_query_failure_is_reported_as_a_query_error` | unit | — | +| `test_rejected_tokens` | — | — | -| `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | +| `test_valid_token_for_a_member_gets_a_session` | — | — | -| `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | +| `test_valid_token_but_no_membership_or_account_is_401` | — | — | -| `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | +| `test_invalid_token_is_401` | — | — | -| `TierXSqliteStructure.test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | unit | — | +| `test_inactive_user_is_401` | — | — | -| `TierXSqliteStructure.test_structure_is_read_back_from_executed_ddl` | unit | — | +| `test_route_is_404_when_oidc_is_not_configured` | — | — | -| `TierXSqliteStructure.test_invalid_ddl_is_reported_rather_than_raised` | unit | — | +| `test_off_by_default_and_complete_config_passes` | — | — | -| `TierXSqliteStructure.test_sqlite_internal_tables_are_excluded` | unit | — | +| `test_missing_audience_or_insecure_jwks_url_refuses_to_start` | — | — | -| `TierXSqliteStructure.test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | unit | — | +## `tests/system/test_original_bytes.py` -| `TierXSqliteStructure.test_known_missing_tables_are_genuinely_absent` | unit | — | +| Test | Markers | Objective | +|---|---|---| +| `OriginalBytesAreKept.test_bom_prefixed_csv_round_trips_byte_identical` | integration | — | -| `TierXSqliteStructure.test_missing_plus_present_accounts_for_all_twelve_tables` | unit | — | +| `OriginalBytesAreKept.test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | integration | — | -| `TierRegistration.test_all_five_tiers_are_registered` | unit | — | +| `OriginalBytesAreKept.test_the_decoded_view_still_feeds_the_loader_and_source_rows` | integration | — | -| `test_identical_schemas_report_no_differences` | — | — | +| `OriginalBytesAreKept.test_a_failed_upload_keeps_the_raw_original_too` | integration | — | -| `test_namespace_is_not_part_of_the_comparison` | — | — | +| `OriginalBytesAreKept.test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | integration | — | -| `test_missing_table_is_caught` | — | — | +| `OriginalBytesAreKept.test_invalid_base64_is_rejected_before_anything_loads` | integration | — | -| `test_extra_table_is_caught` | — | — | +| `test_bom_prefixed_csv_round_trips_byte_identical` | — | — | -| `test_added_column_is_caught` | — | — | +| `test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | — | — | -| `test_changed_type_is_caught` | — | — | +| `test_the_decoded_view_still_feeds_the_loader_and_source_rows` | — | — | -| `test_changed_nullability_is_caught` | — | — | +| `test_a_failed_upload_keeps_the_raw_original_too` | — | — | -| `test_reordered_columns_are_caught_as_drift` | — | — | +| `test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | — | — | -| `test_env_targets_cover_all_four_environments` | — | — | +| `test_invalid_base64_is_rejected_before_anything_loads` | — | — | -| `test_query_is_sent_on_stdin_not_via_dash_c` | — | — | +## `tests/system/test_password_reset.py` -| `test_query_failure_is_reported_as_a_query_error` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `ChangeOwnPassword.test_new_password_works_old_does_not_and_other_sessions_end` | integration | — | -| `test_genuinely_empty_schema_still_fails` | — | — | +| `ChangeOwnPassword.test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | integration | — | -| `test_placeholders_are_substituted` | — | — | +| `ChangeOwnPassword.test_short_new_password_is_rejected` | integration | — | -| `test_every_known_table_has_a_substitution` | — | — | +| `ChangeOwnPassword.test_needs_a_session` | integration | — | -| `test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | — | — | +| `AdminReset.test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | integration | — | -| `test_structure_is_read_back_from_executed_ddl` | — | — | +| `AdminReset.test_reset_is_written_to_the_tenants_audit_log` | integration | — | -| `test_invalid_ddl_is_reported_rather_than_raised` | — | — | +| `AdminReset.test_non_admin_cannot_reset` | integration | — | -| `test_sqlite_internal_tables_are_excluded` | — | — | +| `AdminReset.test_admin_cannot_reset_a_user_in_another_tenant` | integration | — | -| `test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | — | — | +| `AdminReset.test_user_who_is_also_in_another_tenant_is_refused` | integration | — | -| `test_known_missing_tables_are_genuinely_absent` | — | — | +| `BreakGlassReset.test_resets_password_clears_lockout_and_ends_sessions` | integration | — | -| `test_missing_plus_present_accounts_for_all_twelve_tables` | — | — | +| `BreakGlassReset.test_unknown_user_and_short_password_fail_without_changes` | integration | — | -| `test_all_five_tiers_are_registered` | — | — | +| `test_new_password_works_old_does_not_and_other_sessions_end` | — | — | -## `tests/test_failed_uploads.py` +| `test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `EveryRowFailed.test_every_row_failing_is_an_error_and_saves_nothing` | integration | — | +| `test_short_new_password_is_rejected` | — | — | -| `EveryRowFailed.test_failed_overwrite_keeps_the_previous_upload` | integration | — | +| `test_needs_a_session` | — | — | -| `EveryRowFailed.test_partly_failed_upload_still_imports_and_reports_the_failures` | integration | — | +| `test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | — | — | -| `test_every_row_failing_is_an_error_and_saves_nothing` | — | — | +| `test_reset_is_written_to_the_tenants_audit_log` | — | — | -| `test_failed_overwrite_keeps_the_previous_upload` | — | — | +| `test_non_admin_cannot_reset` | — | — | -| `test_partly_failed_upload_still_imports_and_reports_the_failures` | — | — | +| `test_admin_cannot_reset_a_user_in_another_tenant` | — | — | + +| `test_user_who_is_also_in_another_tenant_is_refused` | — | — | + +| `test_resets_password_clears_lockout_and_ends_sessions` | — | — | + +| `test_unknown_user_and_short_password_fail_without_changes` | — | — | -## `tests/test_header_mapping.py` +## `tests/system/test_preserved_uploads.py` | Test | Markers | Objective | |---|---|---| -| `ApplyHeaderMapping.test_no_mapping_returns_headers_unchanged` | unit | — | +| `PreservedUploads.test_original_bytes_and_rows_are_kept_with_the_outcome` | integration | — | -| `ApplyHeaderMapping.test_rename_replaces_only_the_named_header` | unit | — | +| `PreservedUploads.test_a_failed_upload_still_keeps_its_original` | integration | — | -| `ApplyHeaderMapping.test_blank_rename_keeps_the_original` | unit | — | +| `PreservedUploads.test_failure_reasons_and_log_are_readable_from_another_session` | integration | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | -| `ApplyHeaderMapping.test_unknown_original_is_rejected` | unit | — | +| `PreservedUploads.test_partly_failed_and_duplicate_outcomes` | integration | — | -| `ApplyHeaderMapping.test_original_listed_twice_is_rejected` | unit | — | +| `PreservedUploads.test_overwrite_keeps_the_replaced_original` | integration | — | -| `ApplyHeaderMapping.test_rename_colliding_with_an_untouched_column_is_rejected` | unit | — | +| `PreservedUploads.test_preserved_tables_reject_update_and_delete` | integration | — | -| `ApplyHeaderMapping.test_collision_is_judged_after_sanitising` | unit | — | +| `PreservedUploads.test_unknown_upload_is_404` | integration | — | -| `ApplyHeaderMapping.test_swapping_two_names_is_allowed` | unit | — | +| `test_original_bytes_and_rows_are_kept_with_the_outcome` | — | — | -| `HeaderMappingOverTheApi.test_renamed_header_becomes_the_database_column` | integration | — | +| `test_a_failed_upload_still_keeps_its_original` | — | — | -| `HeaderMappingOverTheApi.test_upload_without_a_mapping_keeps_csv_headers` | integration | — | +| `test_failure_reasons_and_log_are_readable_from_another_session` | — | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | -| `HeaderMappingOverTheApi.test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | integration | — | +| `test_partly_failed_and_duplicate_outcomes` | — | — | -| `HeaderMappingOverTheApi.test_rename_that_collides_with_another_column_is_an_error` | integration | — | +| `test_overwrite_keeps_the_replaced_original` | — | — | -| `HeaderMappingOverTheApi.test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | integration | — | +| `test_preserved_tables_reject_update_and_delete` | — | — | -| `test_no_mapping_returns_headers_unchanged` | — | — | +| `test_unknown_upload_is_404` | — | — | -| `test_rename_replaces_only_the_named_header` | — | — | +## `tests/system/test_ragged_rows.py` -| `test_blank_rename_keeps_the_original` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `ExtraCells.test_non_empty_cells_beyond_the_header_are_returned` | unit | — | -| `test_unknown_original_is_rejected` | — | — | +| `ExtraCells.test_trailing_empty_cells_are_not_extra` | unit | — | -| `test_original_listed_twice_is_rejected` | — | — | +| `ExtraCells.test_short_rows_have_no_extra_cells` | unit | — | -| `test_rename_colliding_with_an_untouched_column_is_rejected` | — | — | +| `RaggedRowsOverTheApi.test_dynamic_mode_reports_a_row_wider_than_the_header` | integration | — | -| `test_collision_is_judged_after_sanitising` | — | — | +| `RaggedRowsOverTheApi.test_dynamic_mode_accepts_a_trailing_empty_cell` | integration | — | -| `test_swapping_two_names_is_allowed` | — | — | +| `RaggedRowsOverTheApi.test_te_mode_reports_a_row_wider_than_the_header` | integration | — | -| `test_renamed_header_becomes_the_database_column` | — | — | +| `RaggedRowsOverTheApi.test_multiple_ragged_rows_each_get_a_reason` | integration | Every ragged row gets its own entry in rowErrors with a reason. | -| `test_upload_without_a_mapping_keeps_csv_headers` | — | — | +| `test_non_empty_cells_beyond_the_header_are_returned` | — | — | -| `test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | — | — | +| `test_trailing_empty_cells_are_not_extra` | — | — | -| `test_rename_that_collides_with_another_column_is_an_error` | — | — | +| `test_short_rows_have_no_extra_cells` | — | — | -| `test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | — | — | +| `test_dynamic_mode_reports_a_row_wider_than_the_header` | — | — | -## `tests/test_import_counts.py` +| `test_dynamic_mode_accepts_a_trailing_empty_cell` | — | — | + +| `test_te_mode_reports_a_row_wider_than_the_header` | — | — | + +| `test_multiple_ragged_rows_each_get_a_reason` | — | Every ragged row gets its own entry in rowErrors with a reason. | + +## `tests/system/test_registry_race.py` | Test | Markers | Objective | |---|---|---| -| `ImportCountsArePersisted.test_dynamic_import_persists_all_four_counts` | integration | — | +| `RegistryRace.test_concurrent_identical_uploads_one_wins` | integration | Two threads upload the same content simultaneously. | -| `ImportCountsArePersisted.test_te_import_persists_counts` | integration | — | +| `RegistryRace.test_concurrent_same_name_different_content` | integration | Two threads upload different content under the same filename. | -| `test_dynamic_import_persists_all_four_counts` | — | — | +| `test_concurrent_identical_uploads_one_wins` | — | Two threads upload the same content simultaneously. | -| `test_te_import_persists_counts` | — | — | +| `test_concurrent_same_name_different_content` | — | Two threads upload different content under the same filename. | -## `tests/test_issue_04_multi_file_upload.py` +## `tests/system/test_review.py` | Test | Markers | Objective | |---|---|---| -| `MultipleFileUpload.test_many_files_upload_in_one_session` | integration | Uploading several files in sequence must all succeed. | +| `Review.test_pages_through_every_row_with_validation` | integration | — | -| `MultipleFileUpload.test_every_uploaded_file_is_listed` | integration | All uploads appear in /api/csv/files — this drives 'Migrated files'. | +| `Review.test_validates_rows_beyond_the_reported_error_cap` | integration | — | -| `MultipleFileUpload.test_each_file_is_independently_readable` | integration | Rows from one upload must not leak into another's table. | +| `Review.test_row_decisions_record_actor_and_reason_and_the_latest_wins` | integration | — | -| `MultipleFileUpload.test_narrow_and_wide_files_both_load` | integration | One column and many columns must both be accepted. | +| `Review.test_cell_edits_are_versions_over_the_unchanged_original` | integration | — | -| `MultipleFileUpload.test_files_with_different_schemas_coexist` | integration | Unrelated column sets must not collide — no shared fixed schema. | +| `Review.test_refusals` | integration | — | -| `MultipleFileUpload.test_a_larger_file_loads_completely` | integration | Row count is not silently truncated. | +| `Review.test_te_review_shows_the_database_reason_a_row_was_rejected` | integration | — | -| `test_many_files_upload_in_one_session` | — | Uploading several files in sequence must all succeed. | +| `Review.test_changes_are_audited_and_insert_only` | integration | — | -| `test_every_uploaded_file_is_listed` | — | All uploads appear in /api/csv/files — this drives 'Migrated files'. | +| `test_pages_through_every_row_with_validation` | — | — | -| `test_each_file_is_independently_readable` | — | Rows from one upload must not leak into another's table. | +| `test_validates_rows_beyond_the_reported_error_cap` | — | — | -| `test_narrow_and_wide_files_both_load` | — | One column and many columns must both be accepted. | +| `test_row_decisions_record_actor_and_reason_and_the_latest_wins` | — | — | -| `test_files_with_different_schemas_coexist` | — | Unrelated column sets must not collide — no shared fixed schema. | +| `test_cell_edits_are_versions_over_the_unchanged_original` | — | — | -| `test_a_larger_file_loads_completely` | — | Row count is not silently truncated. | +| `test_refusals` | — | — | + +| `test_te_review_shows_the_database_reason_a_row_was_rejected` | — | — | -## `tests/test_issue_05_import_summary.py` +| `test_changes_are_audited_and_insert_only` | — | — | + +## `tests/system/test_review_cases.py` | Test | Markers | Objective | |---|---|---| -| `ImportSummaryCounts.test_all_four_counts_are_returned` | integration | — | - -| `ImportSummaryCounts.test_total_rows_counts_data_rows_not_the_header` | integration | — | +| `ReviewCases.test_P02_paging_bounds` | integration | — | -| `ImportSummaryCounts.test_created_count_matches_rows_actually_inserted` | integration | — | +| `ReviewCases.test_P04_ragged_row_is_flagged` | integration | — | -| `ImportSummaryCounts.test_duplicate_count_reflects_repeated_rows_in_the_file` | integration | Two identical rows plus one distinct: 1 duplicate skipped. | +| `ReviewCases.test_P05_a_failed_upload_can_be_reviewed` | integration | — | -| `ImportSummaryCounts.test_failed_count_reflects_rows_that_could_not_be_cast` | integration | With an int8 column, a non-numeric value must count as failed. | +| `ReviewCases.test_P08_an_edit_to_a_bad_value_is_flagged` | integration | — | -| `ImportSummaryCounts.test_failed_rows_are_reported_with_detail` | integration | A count alone is not actionable — the offending row must be named. | +| `ReviewCases.test_P09_edit_by_column_index_and_to_an_empty_value` | integration | — | -| `ImportSummaryCounts.test_counts_reconcile_for_a_clean_file` | integration | — | +| `ReviewCases.test_P10_reason_is_stored_trimmed` | integration | — | -| `ImportSummaryCounts.test_counts_reconcile_for_a_mixed_file` | integration | One good row, one in-file duplicate, one uncastable value. | +| `ReviewCases.test_P11_actor_is_the_signed_in_user` | integration | — | -| `ImportSummaryCounts.test_counts_are_per_batch_not_cumulative` | integration | A second, larger upload must report its own totals, not a running sum. | +| `ReviewCases.test_P12_every_role_can_view_rows_and_history` | integration | — | -| `test_all_four_counts_are_returned` | — | — | +| `ReviewCases.test_N05_row_numbers_outside_the_upload` | integration | — | -| `test_total_rows_counts_data_rows_not_the_header` | — | — | +| `ReviewCases.test_N06_column_index_out_of_range` | integration | — | -| `test_created_count_matches_rows_actually_inserted` | — | — | +| `ReviewCases.test_N07_unknown_upload_on_every_route` | integration | — | -| `test_duplicate_count_reflects_repeated_rows_in_the_file` | — | Two identical rows plus one distinct: 1 duplicate skipped. | +| `ReviewCases.test_N08_N09_uploader_and_auditor_cannot_change` | integration | — | -| `test_failed_count_reflects_rows_that_could_not_be_cast` | — | With an int8 column, a non-numeric value must count as failed. | +| `ReviewCases.test_N10_an_invalid_session_is_refused` | integration | — | -| `test_failed_rows_are_reported_with_detail` | — | A count alone is not actionable — the offending row must be named. | +| `ReviewCases.test_N11_another_tenant_cannot_see_or_change_the_upload` | integration | — | -| `test_counts_reconcile_for_a_clean_file` | — | — | +| `ReviewCases.test_N12_missing_or_wrong_body_fields` | integration | — | -| `test_counts_reconcile_for_a_mixed_file` | — | One good row, one in-file duplicate, one uncastable value. | +| `ReviewCases.test_N13_review_tables_reject_update_delete_and_truncate` | integration | — | -| `test_counts_are_per_batch_not_cumulative` | — | A second, larger upload must report its own totals, not a running sum. | +| `ReviewCases.test_N14_database_rejects_a_blank_reason` | integration | — | -## `tests/test_migrations.py` +| `ReviewCases.test_N16_a_non_numeric_upload_id` | integration | — | -| Test | Markers | Objective | -|---|---|---| -| `MigrationsOnScratchSchema.test_upgrade_head_creates_tables_and_unique_constraints` | integration | — | +| `test_P02_paging_bounds` | — | — | -| `MigrationsOnScratchSchema.test_control_schema_is_created_and_removed` | integration | — | +| `test_P04_ragged_row_is_flagged` | — | — | -| `MigrationsOnScratchSchema.test_downgrade_reverses_each_step` | integration | — | +| `test_P05_a_failed_upload_can_be_reviewed` | — | — | -| `MigrationsOnScratchSchema.test_baseline_adopts_a_fallback_created_schema` | integration | — | +| `test_P08_an_edit_to_a_bad_value_is_flagged` | — | — | -| `MigrationsOnScratchSchema.test_existing_duplicates_stop_the_upgrade_with_remediation` | integration | — | +| `test_P09_edit_by_column_index_and_to_an_empty_value` | — | — | -| `RegistryUniqueness.test_bootstrap_brings_the_schema_to_head` | integration | — | +| `test_P10_reason_is_stored_trimmed` | — | — | -| `RegistryUniqueness.test_duplicate_hash_is_rejected_by_the_database` | integration | — | +| `test_P11_actor_is_the_signed_in_user` | — | — | -| `RegistryUniqueness.test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | integration | — | +| `test_P12_every_role_can_view_rows_and_history` | — | — | -| `RegistryUniqueness.test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | integration | — | +| `test_N05_row_numbers_outside_the_upload` | — | — | -| `test_upgrade_head_creates_tables_and_unique_constraints` | — | — | +| `test_N06_column_index_out_of_range` | — | — | -| `test_control_schema_is_created_and_removed` | — | — | +| `test_N07_unknown_upload_on_every_route` | — | — | -| `test_downgrade_reverses_each_step` | — | — | +| `test_N08_N09_uploader_and_auditor_cannot_change` | — | — | -| `test_baseline_adopts_a_fallback_created_schema` | — | — | +| `test_N10_an_invalid_session_is_refused` | — | — | -| `test_existing_duplicates_stop_the_upgrade_with_remediation` | — | — | +| `test_N11_another_tenant_cannot_see_or_change_the_upload` | — | — | -| `test_bootstrap_brings_the_schema_to_head` | — | — | +| `test_N12_missing_or_wrong_body_fields` | — | — | -| `test_duplicate_hash_is_rejected_by_the_database` | — | — | +| `test_N13_review_tables_reject_update_delete_and_truncate` | — | — | -| `test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | — | — | +| `test_N14_database_rejects_a_blank_reason` | — | — | -| `test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | — | — | +| `test_N16_a_non_numeric_upload_id` | — | — | -## `tests/test_oidc.py` +## `tests/system/test_roles.py` | Test | Markers | Objective | |---|---|---| -| `TokenVerification.test_valid_token_yields_the_username` | unit | — | - -| `TokenVerification.test_rejected_tokens` | unit | — | - -| `OidcSignIn.test_valid_token_for_a_member_gets_a_session` | integration | — | - -| `OidcSignIn.test_valid_token_but_no_membership_or_account_is_401` | integration | — | +| `MatrixIsDocumented.test_docs_table_matches_the_enforced_permissions` | unit | — | -| `OidcSignIn.test_invalid_token_is_401` | integration | — | +| `MatrixIsDocumented.test_every_action_has_a_route_and_only_known_roles` | unit | — | -| `OidcSignIn.test_inactive_user_is_401` | integration | — | +| `RoleMatrixOverTheApi.test_every_role_against_every_route` | integration | — | -| `OidcSignIn.test_route_is_404_when_oidc_is_not_configured` | integration | — | +| `RoleMatrixOverTheApi.test_no_session_requests_are_not_role_checked` | integration | — | -| `ConfigFailsClosed.test_off_by_default_and_complete_config_passes` | unit | — | +| `test_docs_table_matches_the_enforced_permissions` | — | — | -| `ConfigFailsClosed.test_missing_audience_or_insecure_jwks_url_refuses_to_start` | unit | — | +| `test_every_action_has_a_route_and_only_known_roles` | — | — | -| `test_valid_token_yields_the_username` | — | — | +| `test_every_role_against_every_route` | — | — | -| `test_rejected_tokens` | — | — | +| `test_no_session_requests_are_not_role_checked` | — | — | -| `test_valid_token_for_a_member_gets_a_session` | — | — | +## `tests/system/test_runtime_role.py` -| `test_valid_token_but_no_membership_or_account_is_401` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `RuntimeRole.test_script_keeps_the_password_out_of_its_output` | integration | — | -| `test_invalid_token_is_401` | — | — | +| `RuntimeRole.test_the_api_works_as_the_runtime_role` | integration | — | -| `test_inactive_user_is_401` | — | — | +| `RuntimeRole.test_runtime_role_cannot_change_or_unprotect_append_only_tables` | integration | — | -| `test_route_is_404_when_oidc_is_not_configured` | — | — | +| `test_script_keeps_the_password_out_of_its_output` | — | — | -| `test_off_by_default_and_complete_config_passes` | — | — | +| `test_the_api_works_as_the_runtime_role` | — | — | -| `test_missing_audience_or_insecure_jwks_url_refuses_to_start` | — | — | +| `test_runtime_role_cannot_change_or_unprotect_append_only_tables` | — | — | -## `tests/test_original_bytes.py` +## `tests/system/test_te_loader_integration.py` | Test | Markers | Objective | |---|---|---| -| `OriginalBytesAreKept.test_bom_prefixed_csv_round_trips_byte_identical` | integration | — | +| `TeUploadOrganisations.test_valid_te_upload_returns_ok` | integration | A well-formed T&E CSV must return status=ok. | -| `OriginalBytesAreKept.test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | integration | — | +| `TeUploadOrganisations.test_inserted_rows_count_matches_data_rows` | integration | insertedRows must equal the number of data rows supplied. | -| `OriginalBytesAreKept.test_the_decoded_view_still_feeds_the_loader_and_source_rows` | integration | — | +| `TeUploadOrganisations.test_table_name_in_response_is_schema_qualified` | integration | tableName must be schema-qualified (e.g. te_dev.organisations). | -| `OriginalBytesAreKept.test_a_failed_upload_keeps_the_raw_original_too` | integration | — | +| `TeUploadOrganisations.test_rows_are_queryable_after_upload` | integration | Inserted rows must appear in the T&E table row count. | -| `OriginalBytesAreKept.test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | integration | — | +| `TeUploadOrganisations.test_core_te_tables_undisturbed_after_upload` | integration | A T&E upload must not drop or alter the other 11 core tables. | -| `OriginalBytesAreKept.test_invalid_base64_is_rejected_before_anything_loads` | integration | — | +| `TeUploadOrganisations.test_reupload_cleans_prior_registry_entry` | integration | Re-uploading the same file name must not duplicate registry entries. | -| `test_bom_prefixed_csv_round_trips_byte_identical` | — | — | +| `TeUploadOrganisations.test_invalid_target_table_is_rejected` | integration | A targetTable not in TE_TABLES must return status=error. | -| `test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | — | — | +| `TeUploadOrganisations.test_empty_file_is_rejected` | integration | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | -| `test_the_decoded_view_still_feeds_the_loader_and_source_rows` | — | — | +| `TeUploadOrganisations.test_header_only_file_is_rejected` | integration | A header-only CSV (no data rows) must return status=invalid_structure. | -| `test_a_failed_upload_keeps_the_raw_original_too` | — | — | +| `TeUploadOrganisations.test_unknown_columns_are_rejected` | integration | Columns not in the target table must return status=error. | -| `test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | — | — | +| `TeUploadOrganisations.test_missing_target_table_param_is_rejected` | integration | mode=te with no targetTable must return 422 from the API layer. | -| `test_invalid_base64_is_rejected_before_anything_loads` | — | — | +| `DynamicOverwriteOfTeOriginFile.test_dynamic_overwrite_of_te_origin_entry_succeeds` | integration | A dynamic-mode overwrite of a filename previously registered in | -## `tests/test_parity.py` +| `test_valid_te_upload_returns_ok` | — | A well-formed T&E CSV must return status=ok. | -| Test | Markers | Objective | -|---|---|---| -| `TestDevSeedCounts.test_dev_seed_row_counts` | — | — | +| `test_inserted_rows_count_matches_data_rows` | — | insertedRows must equal the number of data rows supplied. | -| `TestAllEnvironmentsHaveRequiredTables.test_dev_has_required_tables` | — | — | +| `test_table_name_in_response_is_schema_qualified` | — | tableName must be schema-qualified (e.g. te_dev.organisations). | -| `TestAllEnvironmentsHaveRequiredTables.test_test_has_required_tables` | — | — | +| `test_rows_are_queryable_after_upload` | — | Inserted rows must appear in the T&E table row count. | -| `TestAllEnvironmentsHaveRequiredTables.test_staging_has_required_tables` | — | — | +| `test_core_te_tables_undisturbed_after_upload` | — | A T&E upload must not drop or alter the other 11 core tables. | -| `TestAllEnvironmentsHaveRequiredTables.test_prod_has_required_tables` | — | — | +| `test_reupload_cleans_prior_registry_entry` | — | Re-uploading the same file name must not duplicate registry entries. | -| `TestNoSeedInStagingAndProd.test_staging_has_no_seed_data` | — | — | +| `test_invalid_target_table_is_rejected` | — | A targetTable not in TE_TABLES must return status=error. | -| `TestNoSeedInStagingAndProd.test_prod_has_no_seed_data` | — | — | +| `test_empty_file_is_rejected` | — | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | -| `TestIdempotentDeployParity.test_dev_row_counts_stable_after_second_deploy` | — | — | +| `test_header_only_file_is_rejected` | — | A header-only CSV (no data rows) must return status=invalid_structure. | -| `test_dev_seed_row_counts` | — | — | +| `test_unknown_columns_are_rejected` | — | Columns not in the target table must return status=error. | -| `test_dev_has_required_tables` | — | — | +| `test_missing_target_table_param_is_rejected` | — | mode=te with no targetTable must return 422 from the API layer. | -| `test_test_has_required_tables` | — | — | +| `test_dynamic_overwrite_of_te_origin_entry_succeeds` | — | A dynamic-mode overwrite of a filename previously registered in | -| `test_staging_has_required_tables` | — | — | +## `tests/system/test_te_reupload_registry.py` -| `test_prod_has_required_tables` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `TeReuploadRegistry.test_all_failed_reupload_keeps_the_earlier_registry_record` | integration | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | -| `test_staging_has_no_seed_data` | — | — | +| `TeReuploadRegistry.test_reupload_that_inserts_rows_still_replaces_the_registry_record` | integration | A T&E re-upload that loads at least one row still replaces the registry record. | -| `test_prod_has_no_seed_data` | — | — | +| `TeReuploadRegistry.test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | integration | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | -| `test_dev_row_counts_stable_after_second_deploy` | — | — | +| `test_all_failed_reupload_keeps_the_earlier_registry_record` | — | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | + +| `test_reupload_that_inserts_rows_still_replaces_the_registry_record` | — | A T&E re-upload that loads at least one row still replaces the registry record. | -## `tests/test_password_reset.py` +| `test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | — | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | + +## `tests/system/test_tenant_isolation.py` | Test | Markers | Objective | |---|---|---| -| `ChangeOwnPassword.test_new_password_works_old_does_not_and_other_sessions_end` | integration | — | +| `CrossTenantIsolation.test_upload_lands_in_the_tenants_own_schema` | integration | — | -| `ChangeOwnPassword.test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | integration | — | +| `CrossTenantIsolation.test_other_tenant_cannot_list_read_or_delete` | integration | — | -| `ChangeOwnPassword.test_short_new_password_is_rejected` | integration | — | +| `CrossTenantIsolation.test_requests_without_a_session_see_no_tenant_data` | integration | — | -| `ChangeOwnPassword.test_needs_a_session` | integration | — | +| `CrossTenantIsolation.test_duplicate_detection_is_per_tenant` | integration | — | -| `AdminReset.test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | integration | — | +| `CrossTenantIsolation.test_audit_log_is_per_tenant` | integration | — | -| `AdminReset.test_reset_is_written_to_the_tenants_audit_log` | integration | — | +| `CrossTenantIsolation.test_te_mode_loads_into_the_tenants_own_te_tables` | integration | — | -| `AdminReset.test_non_admin_cannot_reset` | integration | — | +| `CrossTenantIsolation.test_te_table_counts_are_per_tenant` | integration | — | -| `AdminReset.test_admin_cannot_reset_a_user_in_another_tenant` | integration | — | +| `CrossTenantIsolation.test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | integration | — | -| `AdminReset.test_user_who_is_also_in_another_tenant_is_refused` | integration | — | +| `CrossTenantIsolation.test_invalid_session_is_401_not_a_fallback_to_default` | integration | — | -| `BreakGlassReset.test_resets_password_clears_lockout_and_ends_sessions` | integration | — | +| `CrossTenantIsolation.test_tenant_schema_is_at_head_with_its_own_registry` | integration | — | -| `BreakGlassReset.test_unknown_user_and_short_password_fail_without_changes` | integration | — | +| `CrossTenantIsolation.test_concurrent_requests_never_see_another_tenants_schema` | integration | — | -| `test_new_password_works_old_does_not_and_other_sessions_end` | — | — | +| `CrossTenantIsolation.test_provisioning_is_idempotent` | integration | — | -| `test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | — | — | +| `test_upload_lands_in_the_tenants_own_schema` | — | — | -| `test_short_new_password_is_rejected` | — | — | +| `test_other_tenant_cannot_list_read_or_delete` | — | — | -| `test_needs_a_session` | — | — | +| `test_requests_without_a_session_see_no_tenant_data` | — | — | -| `test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | — | — | +| `test_duplicate_detection_is_per_tenant` | — | — | -| `test_reset_is_written_to_the_tenants_audit_log` | — | — | +| `test_audit_log_is_per_tenant` | — | — | -| `test_non_admin_cannot_reset` | — | — | +| `test_te_mode_loads_into_the_tenants_own_te_tables` | — | — | -| `test_admin_cannot_reset_a_user_in_another_tenant` | — | — | +| `test_te_table_counts_are_per_tenant` | — | — | -| `test_user_who_is_also_in_another_tenant_is_refused` | — | — | +| `test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | — | — | -| `test_resets_password_clears_lockout_and_ends_sessions` | — | — | +| `test_invalid_session_is_401_not_a_fallback_to_default` | — | — | -| `test_unknown_user_and_short_password_fail_without_changes` | — | — | +| `test_tenant_schema_is_at_head_with_its_own_registry` | — | — | + +| `test_concurrent_requests_never_see_another_tenants_schema` | — | — | -## `tests/test_preflight_ps1.py` +| `test_provisioning_is_idempotent` | — | — | + +## `tests/system_integration/test_csv_loader_arbitrary_shapes.py` | Test | Markers | Objective | |---|---|---| -| `test_no_null_coalescing_operator` | regression, unit | `??` / `??=` is PowerShell 7+ only and fails to parse on 5.1. | +| `CsvLoaderArbitraryShapes.test_shape_tiny` | — | — | -| `test_no_other_powershell7_only_syntax` | regression, unit | — | +| `CsvLoaderArbitraryShapes.test_shape_medium` | — | — | -| `test_no_utf8_bom` | regression, unit | A BOM caused a noisy line-1 diff during the fix; keep the file BOM-free. | +| `CsvLoaderArbitraryShapes.test_shape_skinny` | — | — | -| `test_port_label_is_not_hardcoded` | regression, unit | The check label must follow the configured port (was fixed at 'port 5432'). | +| `test_shape_tiny` | — | — | -| `test_connection_defaults_resolved_once_and_reused` | regression, unit | Defaults live in variables; connection commands must use them. | +| `test_shape_medium` | — | — | -| `test_default_port_and_host_unchanged` | regression, unit | — | +| `test_shape_skinny` | — | — | -| `test_exit_code_contract_documented_and_implemented` | regression, unit | — | +## `tests/system_integration/test_e2e_pipeline.py` -| `test_parses_under_windows_powershell` | regression | Parse with the real PowerShell parser (5.1 on stock Windows). | +| Test | Markers | Objective | +|---|---|---| +| `E2EPipelineValidateOnly.test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | -| `test_runs_with_skippg_and_reports_summary` | regression | — | +| `E2EPipelineValidateOnly.test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | -| `test_default_port_label_when_pgport_unset` | regression | — | +| `E2EPipelineValidateOnly.test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | -| `test_port_label_follows_pgport` | regression | — | +| `E2EPipelineValidateOnly.test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | -| `test_pg_checks_are_optional_never_fail` | regression | — | +| `E2EPipelineWithDatabase.test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | -## `tests/test_preserved_uploads.py` +| `E2EPipelineWithDatabase.test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | + +| `test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | + +| `test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | + +| `test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | + +| `test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | + +| `test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | + +| `test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | + +## `tests/system_integration/test_parity.py` | Test | Markers | Objective | |---|---|---| -| `PreservedUploads.test_original_bytes_and_rows_are_kept_with_the_outcome` | integration | — | +| `TestDevSeedCounts.test_dev_seed_row_counts` | — | — | -| `PreservedUploads.test_a_failed_upload_still_keeps_its_original` | integration | — | +| `TestAllEnvironmentsHaveRequiredTables.test_dev_has_required_tables` | — | — | -| `PreservedUploads.test_failure_reasons_and_log_are_readable_from_another_session` | integration | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | +| `TestAllEnvironmentsHaveRequiredTables.test_test_has_required_tables` | — | — | -| `PreservedUploads.test_partly_failed_and_duplicate_outcomes` | integration | — | +| `TestAllEnvironmentsHaveRequiredTables.test_staging_has_required_tables` | — | — | -| `PreservedUploads.test_overwrite_keeps_the_replaced_original` | integration | — | +| `TestAllEnvironmentsHaveRequiredTables.test_prod_has_required_tables` | — | — | -| `PreservedUploads.test_preserved_tables_reject_update_and_delete` | integration | — | +| `TestNoSeedInStagingAndProd.test_staging_has_no_seed_data` | — | — | -| `PreservedUploads.test_unknown_upload_is_404` | integration | — | +| `TestNoSeedInStagingAndProd.test_prod_has_no_seed_data` | — | — | -| `test_original_bytes_and_rows_are_kept_with_the_outcome` | — | — | +| `TestIdempotentDeployParity.test_dev_row_counts_stable_after_second_deploy` | — | — | -| `test_a_failed_upload_still_keeps_its_original` | — | — | +| `test_dev_seed_row_counts` | — | — | -| `test_failure_reasons_and_log_are_readable_from_another_session` | — | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | +| `test_dev_has_required_tables` | — | — | -| `test_partly_failed_and_duplicate_outcomes` | — | — | +| `test_test_has_required_tables` | — | — | -| `test_overwrite_keeps_the_replaced_original` | — | — | +| `test_staging_has_required_tables` | — | — | -| `test_preserved_tables_reject_update_and_delete` | — | — | +| `test_prod_has_required_tables` | — | — | -| `test_unknown_upload_is_404` | — | — | +| `test_staging_has_no_seed_data` | — | — | -## `tests/test_ragged_rows.py` +| `test_prod_has_no_seed_data` | — | — | + +| `test_dev_row_counts_stable_after_second_deploy` | — | — | + +## `tests/unit/test_api_hardening.py` | Test | Markers | Objective | |---|---|---| -| `ExtraCells.test_non_empty_cells_beyond_the_header_are_returned` | unit | — | +| `AuthFailsClosed.test_dev_may_run_without_a_key` | unit | — | -| `ExtraCells.test_trailing_empty_cells_are_not_extra` | unit | — | +| `AuthFailsClosed.test_any_key_is_accepted_in_dev` | unit | — | -| `ExtraCells.test_short_rows_have_no_extra_cells` | unit | — | +| `AuthFailsClosed.test_prod_without_a_key_refuses_to_start` | unit | — | -| `RaggedRowsOverTheApi.test_dynamic_mode_reports_a_row_wider_than_the_header` | integration | — | +| `AuthFailsClosed.test_staging_and_test_also_refuse` | unit | — | -| `RaggedRowsOverTheApi.test_dynamic_mode_accepts_a_trailing_empty_cell` | integration | — | +| `AuthFailsClosed.test_deployed_env_with_a_key_starts` | unit | — | -| `RaggedRowsOverTheApi.test_te_mode_reports_a_row_wider_than_the_header` | integration | — | +| `RateLimiterAlgorithm.test_requests_under_budget_are_allowed` | unit | — | -| `RaggedRowsOverTheApi.test_multiple_ragged_rows_each_get_a_reason` | integration | Every ragged row gets its own entry in rowErrors with a reason. | +| `RateLimiterAlgorithm.test_request_over_budget_is_refused` | unit | — | -| `test_non_empty_cells_beyond_the_header_are_returned` | — | — | +| `RateLimiterAlgorithm.test_retry_after_points_at_the_oldest_hit_leaving_the_window` | unit | — | -| `test_trailing_empty_cells_are_not_extra` | — | — | +| `RateLimiterAlgorithm.test_window_expiry_frees_the_budget` | unit | — | -| `test_short_rows_have_no_extra_cells` | — | — | +| `RateLimiterAlgorithm.test_zero_budget_disables_the_limit` | unit | — | -| `test_dynamic_mode_reports_a_row_wider_than_the_header` | — | — | +| `RateLimiterAlgorithm.test_read_and_write_scopes_are_independent` | unit | — | -| `test_dynamic_mode_accepts_a_trailing_empty_cell` | — | — | +| `RateLimiterAlgorithm.test_clients_are_independent` | unit | — | -| `test_te_mode_reports_a_row_wider_than_the_header` | — | — | +| `RateLimiterAlgorithm.test_bucket_table_is_bounded` | unit | — | -| `test_multiple_ragged_rows_each_get_a_reason` | — | Every ragged row gets its own entry in rowErrors with a reason. | +| `RateLimiterAlgorithm.test_reset_clears_state` | unit | — | + +| `RateLimitMiddleware.test_exceeding_the_write_budget_returns_429_with_retry_after` | unit | — | + +| `RateLimitMiddleware.test_health_is_never_rate_limited` | unit | — | + +| `RateLimitMiddleware.test_openapi_is_never_rate_limited` | unit | — | -## `tests/test_registry_race.py` +| `RateLimitMiddleware.test_disabled_budget_allows_everything` | unit | — | + +| `RateLimitMiddleware.test_forwarded_for_cannot_be_used_to_evade_the_limit` | unit | — | + +| `AuditWriterContract.test_only_known_actions_are_accepted` | unit | — | + +| `AuditWriterContract.test_unknown_action_raises_before_any_sql_runs` | unit | — | + +| `AuditWriterContract.test_valid_action_reaches_the_cursor_with_bound_parameters` | unit | — | + +| `AuditWriterContract.test_file_id_is_coerced_from_the_loaders_string_form` | unit | — | + +| `AuditWriterContract.test_unparseable_file_id_degrades_to_null_rather_than_raising` | unit | — | + +| `test_dev_may_run_without_a_key` | — | — | + +| `test_any_key_is_accepted_in_dev` | — | — | + +| `test_prod_without_a_key_refuses_to_start` | — | — | + +| `test_staging_and_test_also_refuse` | — | — | + +| `test_deployed_env_with_a_key_starts` | — | — | + +| `test_requests_under_budget_are_allowed` | — | — | + +| `test_request_over_budget_is_refused` | — | — | + +| `test_retry_after_points_at_the_oldest_hit_leaving_the_window` | — | — | + +| `test_window_expiry_frees_the_budget` | — | — | + +| `test_zero_budget_disables_the_limit` | — | — | + +| `test_read_and_write_scopes_are_independent` | — | — | + +| `test_clients_are_independent` | — | — | + +| `test_bucket_table_is_bounded` | — | — | + +| `test_reset_clears_state` | — | — | + +| `test_exceeding_the_write_budget_returns_429_with_retry_after` | — | — | + +| `test_health_is_never_rate_limited` | — | — | + +| `test_openapi_is_never_rate_limited` | — | — | + +| `test_disabled_budget_allows_everything` | — | — | + +| `test_forwarded_for_cannot_be_used_to_evade_the_limit` | — | — | + +| `test_only_known_actions_are_accepted` | — | — | + +| `test_unknown_action_raises_before_any_sql_runs` | — | — | + +| `test_valid_action_reaches_the_cursor_with_bound_parameters` | — | — | + +| `test_file_id_is_coerced_from_the_loaders_string_form` | — | — | + +| `test_unparseable_file_id_degrades_to_null_rather_than_raising` | — | — | + +## `tests/unit/test_api_unit.py` | Test | Markers | Objective | |---|---|---| -| `RegistryRace.test_concurrent_identical_uploads_one_wins` | integration | Two threads upload the same content simultaneously. | +| `UnitCsvParse.test_sanitize_columns_no_collision_on_stem_conflict` | unit | BUG-036 regression: ["foo","foo","foo_2"] must produce three | -| `RegistryRace.test_concurrent_same_name_different_content` | integration | Two threads upload different content under the same filename. | +| `UnitCsvParse.test_sanitize_columns_dedupes_true_duplicates` | unit | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | -| `test_concurrent_identical_uploads_one_wins` | — | Two threads upload the same content simultaneously. | +| `UnitCsvParse.test_sanitize_columns_reserved_stems_get_suffix` | unit | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | -| `test_concurrent_same_name_different_content` | — | Two threads upload different content under the same filename. | +| `RequestValidation.test_preview_rejects_empty_filename` | unit | — | + +| `RequestValidation.test_preview_rejects_empty_content` | unit | — | + +| `RequestValidation.test_preview_rejects_overlong_filename` | unit | — | + +| `RequestValidation.test_upload_rejects_unknown_mode` | unit | — | + +| `RequestValidation.test_upload_te_mode_requires_target_table` | unit | — | + +| `RequestValidation.test_oversized_upload_is_rejected` | unit | — | -## `tests/test_regression.py` +| `TableNameGuards.test_non_upload_tables_are_rejected` | security, unit | — | + +| `TableNameGuards.test_overlong_names_are_rejected` | security, unit | — | + +| `TableNameGuards.test_identifiers_are_never_string_formatted` | security, unit | Every SQL identifier must go through psycopg2.sql.Identifier. | + +| `test_sanitize_columns_no_collision_on_stem_conflict` | — | BUG-036 regression: ["foo","foo","foo_2"] must produce three | + +| `test_sanitize_columns_dedupes_true_duplicates` | — | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | + +| `test_sanitize_columns_reserved_stems_get_suffix` | — | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | + +| `test_preview_rejects_empty_filename` | — | — | + +| `test_preview_rejects_empty_content` | — | — | + +| `test_preview_rejects_overlong_filename` | — | — | + +| `test_upload_rejects_unknown_mode` | — | — | + +| `test_upload_te_mode_requires_target_table` | — | — | + +| `test_oversized_upload_is_rejected` | — | — | + +| `test_non_upload_tables_are_rejected` | — | — | + +| `test_overlong_names_are_rejected` | — | — | + +| `test_identifiers_are_never_string_formatted` | — | Every SQL identifier must go through psycopg2.sql.Identifier. | + +## `tests/unit/test_ci_quality_gate.py` | Test | Markers | Objective | |---|---|---| -| `RegressionBomHandling.test_bom_csv_is_accepted` | — | — | +| `TestQualityGateStructure.test_has_free_tier_job` | — | — | -| `RegressionCrlfLineEndings.test_crlf_csv_accepted` | — | — | +| `TestQualityGateStructure.test_has_integration_postgres_job` | — | — | -| `RegressionHeaderOnlyCsv.test_header_only_exits_cleanly` | — | — | +| `TestQualityGateStructure.test_triggers_on_pull_request` | — | — | -| `RegressionSingleColumnCsv.test_single_column_valid_rows_pass` | — | — | +| `TestQualityGateStructure.test_triggers_on_push_to_main` | — | — | -| `RegressionNoTraceback.test_missing_env_vars_no_traceback` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_checkout` | — | — | -| `RegressionNoTraceback.test_nonexistent_file_no_traceback` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_python_setup` | — | — | -| `RegressionNoTraceback.test_invalid_utf8_no_traceback` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_artifact_upload` | — | — | -| `test_bom_csv_is_accepted` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_install_dev_deps` | — | — | -| `test_crlf_csv_accepted` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_row_exists` | — | — | -| `test_header_only_exits_cleanly` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_is_accepted_or_deferred` | — | — | -| `test_single_column_valid_rows_pass` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_section_exists` | — | — | -| `test_missing_env_vars_no_traceback` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_is_not_marked_closed` | — | — | -| `test_nonexistent_file_no_traceback` | — | — | +| `test_has_free_tier_job` | — | — | -| `test_invalid_utf8_no_traceback` | — | — | +| `test_has_integration_postgres_job` | — | — | + +| `test_triggers_on_pull_request` | — | — | + +| `test_triggers_on_push_to_main` | — | — | + +| `test_all_jobs_have_checkout` | — | — | + +| `test_all_jobs_have_python_setup` | — | — | + +| `test_all_jobs_have_artifact_upload` | — | — | + +| `test_all_jobs_install_dev_deps` | — | — | + +| `test_g2_row_exists` | — | — | + +| `test_g2_is_accepted_or_deferred` | — | — | + +| `test_g2_section_exists` | — | — | + +| `test_g2_is_not_marked_closed` | — | — | + +## `tests/unit/test_csv_utilise.py` + +| Test | Markers | Objective | +|---|---|---| +| `CsvUtiliseArgumentParsing.test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | + +| `CsvUtiliseArgumentParsing.test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | + +| `CsvUtiliseArgumentParsing.test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | + +| `CsvUtiliseArgumentParsing.test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | + +| `CsvUtiliseArgumentParsing.test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | + +| `CsvUtiliseArgumentParsing.test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | + +| `CsvUtiliseArgumentParsing.test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | + +| `CsvUtiliseArgumentParsing.test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | + +| `CsvUtiliseArgumentParsing.test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | + +| `test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | + +| `test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | + +| `test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | + +| `test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | + +| `test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | + +| `test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | + +| `test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | + +| `test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | + +| `test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | + +## `tests/unit/test_csv_validator.py` + +| Test | Markers | Objective | +|---|---|---| +| `CsvValidatorTests.test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | + +| `CsvValidatorTests.test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | + +| `CsvValidatorTests.test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | + +| `CsvValidatorTests.test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | + +| `test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | + +| `test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | + +| `test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | + +| `test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | + +## `tests/unit/test_evals_runner.py` + +| Test | Markers | Objective | +|---|---|---| +| `EvalsRunnerTests.test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | + +| `EvalsRunnerTests.test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | + +| `EvalsRunnerTests.test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | + +| `EvalsRunnerTests.test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | + +| `EvalsRunnerTests.test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | + +| `EvalsRunnerTests.test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | + +| `EvalsRunnerTests.test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | + +| `test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | + +| `test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | + +| `test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | + +| `test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | + +| `test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | + +| `test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | + +| `test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | + +## `tests/unit/test_evals_tier_e_x.py` + +| Test | Markers | Objective | +|---|---|---| +| `TierEDiffSignatures.test_identical_schemas_report_no_differences` | unit | — | + +| `TierEDiffSignatures.test_namespace_is_not_part_of_the_comparison` | unit | — | + +| `TierEDiffSignatures.test_missing_table_is_caught` | unit | — | + +| `TierEDiffSignatures.test_extra_table_is_caught` | unit | — | + +| `TierEDiffSignatures.test_added_column_is_caught` | unit | — | -## `tests/test_review.py` +| `TierEDiffSignatures.test_changed_type_is_caught` | unit | — | -| Test | Markers | Objective | -|---|---|---| -| `Review.test_pages_through_every_row_with_validation` | integration | — | +| `TierEDiffSignatures.test_changed_nullability_is_caught` | unit | — | -| `Review.test_validates_rows_beyond_the_reported_error_cap` | integration | — | +| `TierEDiffSignatures.test_reordered_columns_are_caught_as_drift` | unit | — | -| `Review.test_row_decisions_record_actor_and_reason_and_the_latest_wins` | integration | — | +| `TierEDiffSignatures.test_env_targets_cover_all_four_environments` | unit | — | -| `Review.test_cell_edits_are_versions_over_the_unchanged_original` | integration | — | +| `TierEColumnSignatureQuery.test_query_is_sent_on_stdin_not_via_dash_c` | unit | — | -| `Review.test_refusals` | integration | — | +| `TierEColumnSignatureQuery.test_query_failure_is_reported_as_a_query_error` | unit | — | -| `Review.test_te_review_shows_the_database_reason_a_row_was_rejected` | integration | — | +| `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | -| `Review.test_changes_are_audited_and_insert_only` | integration | — | +| `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | -| `test_pages_through_every_row_with_validation` | — | — | +| `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | -| `test_validates_rows_beyond_the_reported_error_cap` | — | — | +| `TierXSqliteStructure.test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | unit | — | -| `test_row_decisions_record_actor_and_reason_and_the_latest_wins` | — | — | +| `TierXSqliteStructure.test_structure_is_read_back_from_executed_ddl` | unit | — | -| `test_cell_edits_are_versions_over_the_unchanged_original` | — | — | +| `TierXSqliteStructure.test_invalid_ddl_is_reported_rather_than_raised` | unit | — | -| `test_refusals` | — | — | +| `TierXSqliteStructure.test_sqlite_internal_tables_are_excluded` | unit | — | -| `test_te_review_shows_the_database_reason_a_row_was_rejected` | — | — | +| `TierXSqliteStructure.test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | unit | — | -| `test_changes_are_audited_and_insert_only` | — | — | +| `TierXSqliteStructure.test_known_missing_tables_are_genuinely_absent` | unit | — | -## `tests/test_review_cases.py` +| `TierXSqliteStructure.test_missing_plus_present_accounts_for_all_twelve_tables` | unit | — | -| Test | Markers | Objective | -|---|---|---| -| `ReviewCases.test_P02_paging_bounds` | integration | — | +| `TierRegistration.test_all_five_tiers_are_registered` | unit | — | -| `ReviewCases.test_P04_ragged_row_is_flagged` | integration | — | +| `test_identical_schemas_report_no_differences` | — | — | -| `ReviewCases.test_P05_a_failed_upload_can_be_reviewed` | integration | — | +| `test_namespace_is_not_part_of_the_comparison` | — | — | -| `ReviewCases.test_P08_an_edit_to_a_bad_value_is_flagged` | integration | — | +| `test_missing_table_is_caught` | — | — | -| `ReviewCases.test_P09_edit_by_column_index_and_to_an_empty_value` | integration | — | +| `test_extra_table_is_caught` | — | — | -| `ReviewCases.test_P10_reason_is_stored_trimmed` | integration | — | +| `test_added_column_is_caught` | — | — | -| `ReviewCases.test_P11_actor_is_the_signed_in_user` | integration | — | +| `test_changed_type_is_caught` | — | — | -| `ReviewCases.test_P12_every_role_can_view_rows_and_history` | integration | — | +| `test_changed_nullability_is_caught` | — | — | -| `ReviewCases.test_N05_row_numbers_outside_the_upload` | integration | — | +| `test_reordered_columns_are_caught_as_drift` | — | — | -| `ReviewCases.test_N06_column_index_out_of_range` | integration | — | +| `test_env_targets_cover_all_four_environments` | — | — | -| `ReviewCases.test_N07_unknown_upload_on_every_route` | integration | — | +| `test_query_is_sent_on_stdin_not_via_dash_c` | — | — | -| `ReviewCases.test_N08_N09_uploader_and_auditor_cannot_change` | integration | — | +| `test_query_failure_is_reported_as_a_query_error` | — | — | -| `ReviewCases.test_N10_an_invalid_session_is_refused` | integration | — | +| `test_genuinely_empty_schema_still_fails` | — | — | -| `ReviewCases.test_N11_another_tenant_cannot_see_or_change_the_upload` | integration | — | +| `test_placeholders_are_substituted` | — | — | -| `ReviewCases.test_N12_missing_or_wrong_body_fields` | integration | — | +| `test_every_known_table_has_a_substitution` | — | — | -| `ReviewCases.test_N13_review_tables_reject_update_delete_and_truncate` | integration | — | +| `test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | — | — | -| `ReviewCases.test_N14_database_rejects_a_blank_reason` | integration | — | +| `test_structure_is_read_back_from_executed_ddl` | — | — | -| `ReviewCases.test_N16_a_non_numeric_upload_id` | integration | — | +| `test_invalid_ddl_is_reported_rather_than_raised` | — | — | -| `test_P02_paging_bounds` | — | — | +| `test_sqlite_internal_tables_are_excluded` | — | — | -| `test_P04_ragged_row_is_flagged` | — | — | +| `test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | — | — | -| `test_P05_a_failed_upload_can_be_reviewed` | — | — | +| `test_known_missing_tables_are_genuinely_absent` | — | — | -| `test_P08_an_edit_to_a_bad_value_is_flagged` | — | — | +| `test_missing_plus_present_accounts_for_all_twelve_tables` | — | — | -| `test_P09_edit_by_column_index_and_to_an_empty_value` | — | — | +| `test_all_five_tiers_are_registered` | — | — | -| `test_P10_reason_is_stored_trimmed` | — | — | +## `tests/unit/test_preflight_ps1.py` -| `test_P11_actor_is_the_signed_in_user` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `test_no_null_coalescing_operator` | regression, unit | `??` / `??=` is PowerShell 7+ only and fails to parse on 5.1. | -| `test_P12_every_role_can_view_rows_and_history` | — | — | +| `test_no_other_powershell7_only_syntax` | regression, unit | — | -| `test_N05_row_numbers_outside_the_upload` | — | — | +| `test_no_utf8_bom` | regression, unit | A BOM caused a noisy line-1 diff during the fix; keep the file BOM-free. | -| `test_N06_column_index_out_of_range` | — | — | +| `test_port_label_is_not_hardcoded` | regression, unit | The check label must follow the configured port (was fixed at 'port 5432'). | -| `test_N07_unknown_upload_on_every_route` | — | — | +| `test_connection_defaults_resolved_once_and_reused` | regression, unit | Defaults live in variables; connection commands must use them. | -| `test_N08_N09_uploader_and_auditor_cannot_change` | — | — | +| `test_default_port_and_host_unchanged` | regression, unit | — | -| `test_N10_an_invalid_session_is_refused` | — | — | +| `test_exit_code_contract_documented_and_implemented` | regression, unit | — | -| `test_N11_another_tenant_cannot_see_or_change_the_upload` | — | — | +| `test_parses_under_windows_powershell` | regression | Parse with the real PowerShell parser (5.1 on stock Windows). | -| `test_N12_missing_or_wrong_body_fields` | — | — | +| `test_runs_with_skippg_and_reports_summary` | regression | — | -| `test_N13_review_tables_reject_update_delete_and_truncate` | — | — | +| `test_default_port_label_when_pgport_unset` | regression | — | -| `test_N14_database_rejects_a_blank_reason` | — | — | +| `test_port_label_follows_pgport` | regression | — | -| `test_N16_a_non_numeric_upload_id` | — | — | +| `test_pg_checks_are_optional_never_fail` | regression | — | -## `tests/test_roles.py` +## `tests/unit/test_regression.py` | Test | Markers | Objective | |---|---|---| -| `MatrixIsDocumented.test_docs_table_matches_the_enforced_permissions` | unit | — | - -| `MatrixIsDocumented.test_every_action_has_a_route_and_only_known_roles` | unit | — | +| `RegressionBomHandling.test_bom_csv_is_accepted` | — | — | -| `RoleMatrixOverTheApi.test_every_role_against_every_route` | integration | — | +| `RegressionCrlfLineEndings.test_crlf_csv_accepted` | — | — | -| `RoleMatrixOverTheApi.test_no_session_requests_are_not_role_checked` | integration | — | +| `RegressionHeaderOnlyCsv.test_header_only_exits_cleanly` | — | — | -| `test_docs_table_matches_the_enforced_permissions` | — | — | +| `RegressionSingleColumnCsv.test_single_column_valid_rows_pass` | — | — | -| `test_every_action_has_a_route_and_only_known_roles` | — | — | +| `RegressionNoTraceback.test_missing_env_vars_no_traceback` | — | — | -| `test_every_role_against_every_route` | — | — | +| `RegressionNoTraceback.test_nonexistent_file_no_traceback` | — | — | -| `test_no_session_requests_are_not_role_checked` | — | — | +| `RegressionNoTraceback.test_invalid_utf8_no_traceback` | — | — | -## `tests/test_runtime_role.py` +| `test_bom_csv_is_accepted` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `RuntimeRole.test_script_keeps_the_password_out_of_its_output` | integration | — | +| `test_crlf_csv_accepted` | — | — | -| `RuntimeRole.test_the_api_works_as_the_runtime_role` | integration | — | +| `test_header_only_exits_cleanly` | — | — | -| `RuntimeRole.test_runtime_role_cannot_change_or_unprotect_append_only_tables` | integration | — | +| `test_single_column_valid_rows_pass` | — | — | -| `test_script_keeps_the_password_out_of_its_output` | — | — | +| `test_missing_env_vars_no_traceback` | — | — | -| `test_the_api_works_as_the_runtime_role` | — | — | +| `test_nonexistent_file_no_traceback` | — | — | -| `test_runtime_role_cannot_change_or_unprotect_append_only_tables` | — | — | +| `test_invalid_utf8_no_traceback` | — | — | -## `tests/test_security.py` +## `tests/unit/test_security.py` | Test | Markers | Objective | |---|---|---| @@ -1473,7 +1593,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_env_sql_files_no_hardcoded_passwords` | — | — | -## `tests/test_service_loader.py` +## `tests/unit/test_service_loader.py` | Test | Markers | Objective | |---|---|---| @@ -1485,7 +1605,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_te_cleanup_prior_registry_entries_drops_dynamic_table` | — | — | -## `tests/test_snapshot.py` +## `tests/unit/test_snapshot.py` | Test | Markers | Objective | |---|---|---| @@ -1501,123 +1621,3 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_all_valid_no_skips` | — | — | -## `tests/test_te_loader_integration.py` - -| Test | Markers | Objective | -|---|---|---| -| `TeUploadOrganisations.test_valid_te_upload_returns_ok` | integration | A well-formed T&E CSV must return status=ok. | - -| `TeUploadOrganisations.test_inserted_rows_count_matches_data_rows` | integration | insertedRows must equal the number of data rows supplied. | - -| `TeUploadOrganisations.test_table_name_in_response_is_schema_qualified` | integration | tableName must be schema-qualified (e.g. te_dev.organisations). | - -| `TeUploadOrganisations.test_rows_are_queryable_after_upload` | integration | Inserted rows must appear in the T&E table row count. | - -| `TeUploadOrganisations.test_core_te_tables_undisturbed_after_upload` | integration | A T&E upload must not drop or alter the other 11 core tables. | - -| `TeUploadOrganisations.test_reupload_cleans_prior_registry_entry` | integration | Re-uploading the same file name must not duplicate registry entries. | - -| `TeUploadOrganisations.test_invalid_target_table_is_rejected` | integration | A targetTable not in TE_TABLES must return status=error. | - -| `TeUploadOrganisations.test_empty_file_is_rejected` | integration | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | - -| `TeUploadOrganisations.test_header_only_file_is_rejected` | integration | A header-only CSV (no data rows) must return status=invalid_structure. | - -| `TeUploadOrganisations.test_unknown_columns_are_rejected` | integration | Columns not in the target table must return status=error. | - -| `TeUploadOrganisations.test_missing_target_table_param_is_rejected` | integration | mode=te with no targetTable must return 422 from the API layer. | - -| `DynamicOverwriteOfTeOriginFile.test_dynamic_overwrite_of_te_origin_entry_succeeds` | integration | A dynamic-mode overwrite of a filename previously registered in | - -| `test_valid_te_upload_returns_ok` | — | A well-formed T&E CSV must return status=ok. | - -| `test_inserted_rows_count_matches_data_rows` | — | insertedRows must equal the number of data rows supplied. | - -| `test_table_name_in_response_is_schema_qualified` | — | tableName must be schema-qualified (e.g. te_dev.organisations). | - -| `test_rows_are_queryable_after_upload` | — | Inserted rows must appear in the T&E table row count. | - -| `test_core_te_tables_undisturbed_after_upload` | — | A T&E upload must not drop or alter the other 11 core tables. | - -| `test_reupload_cleans_prior_registry_entry` | — | Re-uploading the same file name must not duplicate registry entries. | - -| `test_invalid_target_table_is_rejected` | — | A targetTable not in TE_TABLES must return status=error. | - -| `test_empty_file_is_rejected` | — | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | - -| `test_header_only_file_is_rejected` | — | A header-only CSV (no data rows) must return status=invalid_structure. | - -| `test_unknown_columns_are_rejected` | — | Columns not in the target table must return status=error. | - -| `test_missing_target_table_param_is_rejected` | — | mode=te with no targetTable must return 422 from the API layer. | - -| `test_dynamic_overwrite_of_te_origin_entry_succeeds` | — | A dynamic-mode overwrite of a filename previously registered in | - -## `tests/test_te_reupload_registry.py` - -| Test | Markers | Objective | -|---|---|---| -| `TeReuploadRegistry.test_all_failed_reupload_keeps_the_earlier_registry_record` | integration | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | - -| `TeReuploadRegistry.test_reupload_that_inserts_rows_still_replaces_the_registry_record` | integration | A T&E re-upload that loads at least one row still replaces the registry record. | - -| `TeReuploadRegistry.test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | integration | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | - -| `test_all_failed_reupload_keeps_the_earlier_registry_record` | — | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | - -| `test_reupload_that_inserts_rows_still_replaces_the_registry_record` | — | A T&E re-upload that loads at least one row still replaces the registry record. | - -| `test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | — | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | - -## `tests/test_tenant_isolation.py` - -| Test | Markers | Objective | -|---|---|---| -| `CrossTenantIsolation.test_upload_lands_in_the_tenants_own_schema` | integration | — | - -| `CrossTenantIsolation.test_other_tenant_cannot_list_read_or_delete` | integration | — | - -| `CrossTenantIsolation.test_requests_without_a_session_see_no_tenant_data` | integration | — | - -| `CrossTenantIsolation.test_duplicate_detection_is_per_tenant` | integration | — | - -| `CrossTenantIsolation.test_audit_log_is_per_tenant` | integration | — | - -| `CrossTenantIsolation.test_te_mode_loads_into_the_tenants_own_te_tables` | integration | — | - -| `CrossTenantIsolation.test_te_table_counts_are_per_tenant` | integration | — | - -| `CrossTenantIsolation.test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | integration | — | - -| `CrossTenantIsolation.test_invalid_session_is_401_not_a_fallback_to_default` | integration | — | - -| `CrossTenantIsolation.test_tenant_schema_is_at_head_with_its_own_registry` | integration | — | - -| `CrossTenantIsolation.test_concurrent_requests_never_see_another_tenants_schema` | integration | — | - -| `CrossTenantIsolation.test_provisioning_is_idempotent` | integration | — | - -| `test_upload_lands_in_the_tenants_own_schema` | — | — | - -| `test_other_tenant_cannot_list_read_or_delete` | — | — | - -| `test_requests_without_a_session_see_no_tenant_data` | — | — | - -| `test_duplicate_detection_is_per_tenant` | — | — | - -| `test_audit_log_is_per_tenant` | — | — | - -| `test_te_mode_loads_into_the_tenants_own_te_tables` | — | — | - -| `test_te_table_counts_are_per_tenant` | — | — | - -| `test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | — | — | - -| `test_invalid_session_is_401_not_a_fallback_to_default` | — | — | - -| `test_tenant_schema_is_at_head_with_its_own_registry` | — | — | - -| `test_concurrent_requests_never_see_another_tenants_schema` | — | — | - -| `test_provisioning_is_idempotent` | — | — | - diff --git a/evals/PLAN.md b/evals/PLAN.md index fcff512..f8f45dc 100644 --- a/evals/PLAN.md +++ b/evals/PLAN.md @@ -15,7 +15,7 @@ > Current tier registrations live in `runner.TIER_RUNNERS`: `p, i, s, e, x`. -This folder holds **data-driven evals** for the T&E Database Framework. Evals complement (but don't replace) the existing SQL test suites under `tests/suites/` and the Python unit tests under `tests/test_csv_validator.py`. +This folder holds **data-driven evals** for the T&E Database Framework. Evals complement (but don't replace) the existing SQL test suites under `tests/suites/` and the Python unit tests under `tests/unit/test_csv_validator.py`. ## Why a separate `evals/` folder diff --git a/pytest.ini b/pytest.ini index 49bc7cc..35c3989 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,11 @@ markers = security: static security and credential pattern scans snapshot: golden-file output comparison tests parity: cross-environment row-count consistency tests + system: the API against a live PostgreSQL (tests/system/); added automatically by folder + sit: system integration, whole-pipeline and cross-environment tests (tests/system_integration/); added automatically by folder + performance: load and timing tests (tests/performance/); added automatically by folder + accessibility: WCAG scans in a real browser (tests/accessibility/); added automatically by folder, runs only in its own CI job + uat: user-acceptance scenarios (tests/uat/); added automatically by folder addopts = -ra -q testpaths = tests diff --git a/scripts/health_check.py b/scripts/health_check.py index e68d0d4..2a4eede 100644 --- a/scripts/health_check.py +++ b/scripts/health_check.py @@ -173,11 +173,12 @@ def check_test_infra() -> None: for label, path in files.items(): _check_file(label, path) - test_files = list((ROOT / "tests").glob("test_*.py")) + # Recursive: tests live in category folders (unit/, system/, ...), see tests/README.md. + test_files = list((ROOT / "tests").rglob("test_*.py")) if test_files: _pass(f"python test files ({len(test_files)} found)") else: - _fail("python test files", "no test_*.py found in tests/") + _fail("python test files", "no test_*.py found under tests/") # ── Config template ─────────────────────────────────────────────────────────── diff --git a/scripts/lint.sh b/scripts/lint.sh index 0d15b62..64cca6d 100644 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -27,22 +27,9 @@ PY_SRC=( scripts/create_tenant_admin.py ) -PY_TESTS=( - tests/test_csv_validator.py - tests/test_evals_runner.py - tests/test_api_unit.py - tests/test_api_integration.py - tests/test_issue_04_multi_file_upload.py - tests/test_issue_05_import_summary.py - tests/test_api_hardening.py - tests/test_evals_tier_e_x.py - tests/test_migrations.py - tests/test_import_counts.py - tests/test_accounts.py - tests/test_tenant_isolation.py - tests/test_roles.py - tests/test_oidc.py -) +# Every Python file under tests/ (all categories, conftest, helpers). A hand-kept list had +# fallen behind: it named 14 of the 38 test files (#167). +mapfile -t PY_TESTS < <(find tests -name '*.py' -not -path '*/__pycache__/*' | sort) # Collect scripts/*.py dynamically mapfile -t PY_SCRIPTS < <(ls scripts/*.py 2>/dev/null || true) diff --git a/scripts/provision_full_test_env.sh b/scripts/provision_full_test_env.sh index 2a0f279..53a9fc5 100644 --- a/scripts/provision_full_test_env.sh +++ b/scripts/provision_full_test_env.sh @@ -20,7 +20,7 @@ # Note on passwords: the generated env_.sql keep the literal # '__INJECT_AT_DEPLOY__' placeholder from the template. It is a valid role # password for a local test instance and keeps -# tests/test_security.py::test_env_sql_files_no_hardcoded_passwords green. +# tests/unit/test_security.py::test_env_sql_files_no_hardcoded_passwords green. # Never use these files against a real environment. # ============================================================================= diff --git a/scripts/run_qa.ps1 b/scripts/run_qa.ps1 index 01c18c4..c1cf60d 100644 --- a/scripts/run_qa.ps1 +++ b/scripts/run_qa.ps1 @@ -42,8 +42,8 @@ switch ($Target) { "build/csv/validator.py", "evals/runner.py", "evals/gap_report.py", - "tests/test_csv_validator.py", - "tests/test_evals_runner.py" + "tests/unit/test_csv_validator.py", + "tests/unit/test_evals_runner.py" ) $scriptPy = Get-ChildItem scripts/*.py -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName } diff --git a/scripts/select_tests.py b/scripts/select_tests.py index 60738a9..1914e46 100644 --- a/scripts/select_tests.py +++ b/scripts/select_tests.py @@ -32,7 +32,10 @@ ("build/environments/", "integration or e2e or parity", "env SQL changed → integration + parity"), ("build/schema/", "integration or e2e or parity", "schema changed → integration + parity"), ("tests/suites/", "integration", "SQL test suite changed → integration"), - ("tests/test_", "unit", "test file changed → unit"), + ("tests/unit/", "unit or regression or security or snapshot", "unit test changed → free tier"), + ("tests/system/", "integration", "system test changed → integration"), + ("tests/system_integration/", "e2e or integration or parity", "system-integration test changed → e2e + integration + parity"), + ("tests/accessibility/", "accessibility", "accessibility test changed → accessibility"), ("scripts/", "security", "scripts changed → security scan"), ("build/adapters/", "security", "adapter changed → security scan"), ("build/csv/loader_", "e2e", "loader changed → e2e"), diff --git a/scripts/test_catalogue.py b/scripts/test_catalogue.py index d25a0e5..5019f44 100755 --- a/scripts/test_catalogue.py +++ b/scripts/test_catalogue.py @@ -23,7 +23,8 @@ OUTPUT = ROOT / "docs" / "TEST_CASES.md" MARKER_NAMES = {"unit", "integration", "e2e", "regression", "security", - "snapshot", "parity"} + "snapshot", "parity", "system", "sit", "performance", + "accessibility", "uat"} def _extract_markers(decorators: list[ast.expr]) -> list[str]: diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..ac120c2 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,71 @@ +# Tests + +Python tests are grouped by **what they need to run**, one folder per category. The +folder a test lives in is its category; `conftest.py` adds the matching marker +automatically. + +| Folder | Needs | Marker | Run with | +|---|---|---|---| +| `unit/` | Nothing: no database, no network. Includes the regression, security-scan and snapshot tests | `unit` (explicit) | `make test-unit` | +| `system/` | The API and a live PostgreSQL | `system` (automatic) | `make test-system` | +| `system_integration/` | The whole pipeline: shell loaders, every deployed environment, cross-environment parity | `sit` (automatic) | `make test-sit` | +| `performance/` | Load and timing tests (none yet) | `performance` (automatic) | `make test-performance` | +| `accessibility/` | A real browser against the running app, WCAG scans (#168) | `accessibility` (automatic) | `make test-accessibility` | +| `uat/` | User-acceptance scenarios (none yet) | `uat` (automatic) | `make test-uat` | + +Which folder does a new test go in? Ask what is the least it needs: nothing → +`unit/`; the API against PostgreSQL → `system/`; several components or environments +together → `system_integration/`. + +The finer markers still work and are what CI selects on: `integration`, `e2e`, +`regression`, `security`, `snapshot`, `parity`. The free tier (and the Windows job) +runs `unit or regression or security or snapshot`, so a test that needs a database +must not carry `unit`. + +A file is placed by its heaviest need. Six files in `system/` also hold database-free +classes (marked `unit`); they run in the free tier because that selects on the marker, +not the folder. So `-m unit` and `-m system` overlap by those classes. + +## Not Python, so not moved + +`suites/` and `framework/` are the SQL assertion suites (`run_tests.sh` runs them), +`snapshots/` holds golden files, and `run_python_tests.ps1` is the Windows runner. + +## Running + +```bash +make test-unit # one category +python -m pytest tests/system -v # the same, directly +python -m pytest -m "not accessibility" # everything except the browser job +python -m pytest tests/system/test_roles.py # one file +``` + +Database-backed tests **fail, never skip**, when PostgreSQL isn't reachable; start one +first (`scripts/provision_full_test_env.sh`). A category with no tests yet (`make +test-performance`, `make test-uat`) reports that and succeeds. + +## Shared fixtures (`conftest.py`) + +For tests that drive the live API. `unittest.TestCase` classes can't take fixture +arguments, so each fixture also sets an attribute; use them with +`@pytest.mark.usefixtures(...)`: + +| Fixture | Scope | Gives you | +|---|---|---| +| `live_api` | class | the API started with its database pool; `self.client` | +| `destructive_enabled` | class | `API_ALLOW_DESTRUCTIVE` on for the class, restored after | +| `registry_cleanup` | test | `self.registry_cleanup`: append a file name and its registry row is deleted afterwards | +| `te_org_cleanup` | test | `self.te_org_cleanup`: append organisation names inserted into the T&E schema; deleted afterwards | + +`system/test_failed_uploads.py` and one test in `system/test_review.py` use them. The +other 19 files that start the API by hand haven't been converted. + +## Paths + +A test file reaches the repository root with `Path(__file__).resolve().parents[2]` +(`tests//`); `tests/snapshots/` is `parents[1] / "snapshots"`. + +## The catalogue + +`docs/TEST_CASES.md` lists every test and is generated: run +`python3 scripts/test_catalogue.py` after adding or moving tests. CI fails if it is stale. diff --git a/tests/environments/__init__.py b/tests/accessibility/__init__.py similarity index 100% rename from tests/environments/__init__.py rename to tests/accessibility/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py index 073df61..98b286f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,97 @@ def _reset_rate_limiter(): _limiter.reset() +# ── Shared fixtures for tests that need the live API (#167) ─────────────────────────── +# unittest.TestCase classes cannot take fixture arguments, so each fixture also attaches +# its value (`self.client`, `self.registry_cleanup`, ...) to the class or instance. Use +# them with @pytest.mark.usefixtures("live_api", "destructive_enabled") on the class. +# These fail, never skip, when PostgreSQL is not reachable, like the tests themselves. +from fastapi.testclient import TestClient # noqa: E402 +from psycopg2 import sql # noqa: E402 + +from api.db import Conn # noqa: E402 + +_START_HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/PGPASSWORD/" + "PGDATABASE). See scripts/start-api.ps1.") + + +@pytest.fixture(scope="class") +def destructive_enabled(): + """DELETE and overwrite are off unless API_ALLOW_DESTRUCTIVE is set; turn them on for one class.""" + original = _settings.allow_destructive + _settings.allow_destructive = True + yield + _settings.allow_destructive = original + + +@pytest.fixture(scope="class") +def live_api(request): + """The API started with its lifespan, so the database pool is up. ``self.client`` on a class.""" + ctx = TestClient(app) + try: + client = ctx.__enter__() + except Exception as exc: # noqa: BLE001 + raise AssertionError(f"Could not start the API: {type(exc).__name__}: {exc}\n{_START_HELP}") from None + if request.cls is not None: + request.cls.client = client + yield client + ctx.__exit__(None, None, None) + + +@pytest.fixture +def registry_cleanup(request, live_api, destructive_enabled): + """File names the test registered; their csv_files rows (and tables) are deleted afterwards. + + Append names to the list: ``self.registry_cleanup.append(name)``. + """ + names: list[str] = [] + if request.instance is not None: + request.instance.registry_cleanup = names + yield names + for entry in live_api.get("/api/csv/files").json(): + if entry["file_name"] in names: + live_api.delete(f"/api/csv/files/{entry['id']}") + + +@pytest.fixture +def te_org_cleanup(request): + """Names of organisations the test inserted into the T&E schema; deleted afterwards. + + Append names to the list: ``self.te_org_cleanup.append(name)``. + """ + names: list[str] = [] + if request.instance is not None: + request.instance.te_org_cleanup = names + yield names + if names: + with Conn() as conn, conn.cursor() as cur: + cur.execute(sql.SQL("DELETE FROM {}.organisations WHERE name = ANY(%s)").format( + sql.Identifier(_settings.TE_SCHEMA)), (names,)) + conn.commit() + + +# Folder -> marker (#167). The folder a test lives in is its category, so `-m system`, +# `-m "not accessibility"` and the Makefile targets agree with the layout without each +# test having to remember a decorator. `unit` stays explicit: it is what the free tier +# (and the Windows job) select on, and some folders mix it with finer markers. +_FOLDER_MARKERS = { + "system": "system", + "system_integration": "sit", + "performance": "performance", + "accessibility": "accessibility", + "uat": "uat", +} + + +def pytest_collection_modifyitems(items): + for item in items: + for part in Path(str(item.fspath)).parts: + marker = _FOLDER_MARKERS.get(part) + if marker: + item.add_marker(getattr(pytest.mark, marker)) + break + + # Env vars tests are allowed to mutate; restored after every test. _RESTORE_KEYS = ( "CSV_FILE", diff --git a/tests/environments/test_env_dev.py b/tests/environments/test_env_dev.py deleted file mode 100644 index ad541da..0000000 --- a/tests/environments/test_env_dev.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Contract tests for the dev environment.""" -import unittest -import pytest -from tests.environments._contract import EnvironmentContract - -pytestmark = [pytest.mark.integration, pytest.mark.parity] - - -class DevEnvironment(EnvironmentContract, unittest.TestCase): - ENV = "dev"; DB = "te_mgmt_dev"; SCHEMA = "te_dev" - APP_USER = "te_dev_user"; CONN_LIMIT = 10; SEEDED = True diff --git a/tests/environments/test_env_prod.py b/tests/environments/test_env_prod.py deleted file mode 100644 index 71f25e9..0000000 --- a/tests/environments/test_env_prod.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Contract tests for the prod environment.""" -import unittest -import pytest -from tests.environments._contract import EnvironmentContract - -pytestmark = [pytest.mark.integration, pytest.mark.parity] - - -class ProdEnvironment(EnvironmentContract, unittest.TestCase): - ENV = "prod"; DB = "te_mgmt_prod"; SCHEMA = "te_prod" - APP_USER = "te_prod_user"; CONN_LIMIT = 50; SEEDED = False diff --git a/tests/environments/test_env_staging.py b/tests/environments/test_env_staging.py deleted file mode 100644 index aac766a..0000000 --- a/tests/environments/test_env_staging.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Contract tests for the staging environment.""" -import unittest -import pytest -from tests.environments._contract import EnvironmentContract - -pytestmark = [pytest.mark.integration, pytest.mark.parity] - - -class StagingEnvironment(EnvironmentContract, unittest.TestCase): - ENV = "staging"; DB = "te_mgmt_staging"; SCHEMA = "te_staging" - APP_USER = "te_stg_user"; CONN_LIMIT = 25; SEEDED = False diff --git a/tests/environments/test_env_test.py b/tests/environments/test_env_test.py deleted file mode 100644 index 38efb30..0000000 --- a/tests/environments/test_env_test.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Contract tests for the test environment.""" -import unittest -import pytest -from tests.environments._contract import EnvironmentContract - -pytestmark = [pytest.mark.integration, pytest.mark.parity] - - -class TestEnvironment(EnvironmentContract, unittest.TestCase): - ENV = "test"; DB = "te_mgmt_test"; SCHEMA = "te_test" - APP_USER = "te_test_user"; CONN_LIMIT = 15; SEEDED = True diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/system/__init__.py b/tests/system/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_accounts.py b/tests/system/test_accounts.py similarity index 100% rename from tests/test_accounts.py rename to tests/system/test_accounts.py diff --git a/tests/test_api_coverage.py b/tests/system/test_api_coverage.py similarity index 96% rename from tests/test_api_coverage.py rename to tests/system/test_api_coverage.py index 1ac0e1c..7d17ff4 100644 --- a/tests/test_api_coverage.py +++ b/tests/system/test_api_coverage.py @@ -280,7 +280,6 @@ def test_deleted_file_no_longer_listed(self): self.assertNotIn(str(file_id), ids) - # --------------------------------------------------------------------------- # BUG-041: GET /api/audit/log (integration) # --------------------------------------------------------------------------- @@ -419,7 +418,6 @@ def test_te_upload_registers_in_csv_files(self): self.assertTrue(match, "T&E upload should appear in csv_files list") self.assertEqual(match[0]["mode"], "te") - def test_enum_constraint_violation_reported_per_row(self): """An invalid enum value must produce a per-row error, not a 500.""" csv = self._org_csv(f"{self.tag}_enum", org_type="INVALID_ENUM") @@ -427,7 +425,7 @@ def test_enum_constraint_violation_reported_per_row(self): body = r.json() self.assertEqual(body["status"], "error", body) self.assertGreaterEqual(body["failedRows"], 1, - "Invalid enum value should fail at row level, not crash the endpoint") + "Invalid enum value should fail at row level, not crash the endpoint") self.assertIn("rowNumber", body["rowErrors"][0]) def test_null_in_not_null_column_reported_per_row(self): @@ -437,7 +435,7 @@ def test_null_in_not_null_column_reported_per_row(self): body = r.json() self.assertEqual(body["status"], "error", body) self.assertGreaterEqual(body["failedRows"], 1, - "NULL in a NOT NULL column should fail at row level") + "NULL in a NOT NULL column should fail at row level") self.assertIn("rowNumber", body["rowErrors"][0]) def test_unknown_target_table_returns_error(self): @@ -456,7 +454,7 @@ def test_te_reupload_same_file_replaces_registry(self): """A T&E re-upload that loads rows replaces the registry entry. An identical file would collide on the unique name and change nothing - (#215, see tests/test_te_reupload_registry.py). + (#215, see tests/system/test_te_reupload_registry.py). """ r1 = self._upload_te(self._org_csv(f"{self.tag}_reup_1")) self.assertEqual(r1.json()["status"], "ok", r1.json()) @@ -465,7 +463,7 @@ def test_te_reupload_same_file_replaces_registry(self): files = self.client.get("/api/csv/files").json() matches = [f for f in files if f["file_name"] == f"{self.tag}.csv"] self.assertEqual(len(matches), 1, - "Re-upload should replace the registry entry, not create a duplicate") + "Re-upload should replace the registry entry, not create a duplicate") def test_te_client_side_type_validation_reports_column_and_value(self): """BUG-028: a bad value on a client-validatable column (BOOLEAN here) @@ -485,14 +483,14 @@ def test_te_client_side_type_validation_reports_column_and_value(self): self.assertEqual(body["status"], "ok", body) self.assertEqual(body["failedRows"], 1) self.assertEqual(body["insertedRows"], 1, - "The valid row should still land while the bad row is skipped.") + "The valid row should still land while the bad row is skipped.") self.assertTrue(body["rowErrors"], "rowErrors must be populated") err = body["rowErrors"][0] # These two fields are the fingerprint of the client-side path. self.assertEqual(err.get("column"), "is_active", - f"Client-side validator must set column; got {err!r}") + f"Client-side validator must set column; got {err!r}") self.assertEqual(err.get("value"), "NOT_A_BOOL", - f"Client-side validator must set value; got {err!r}") + f"Client-side validator must set value; got {err!r}") self.assertIn("true/false", err["reason"].lower()) def test_te_upload_rejects_over_row_cap(self): @@ -516,7 +514,7 @@ def test_te_upload_rejects_over_row_cap(self): files = self.client.get("/api/csv/files").json() matches = [f for f in files if f["file_name"] == f"{self.tag}.csv"] self.assertFalse(matches, - "An over-cap upload must not create a registry entry.") + "An over-cap upload must not create a registry entry.") finally: cfg.settings.MAX_ROWS = original @@ -532,6 +530,6 @@ def test_delete_blocked_returns_403(self): try: r = client.delete("/api/csv/files/1") self.assertEqual(r.status_code, 403, - "DELETE must return 403 when API_ALLOW_DESTRUCTIVE=false") + "DELETE must return 403 when API_ALLOW_DESTRUCTIVE=false") finally: cfg.settings.allow_destructive = original diff --git a/tests/test_api_integration.py b/tests/system/test_api_integration.py similarity index 100% rename from tests/test_api_integration.py rename to tests/system/test_api_integration.py diff --git a/tests/test_audit_append_only.py b/tests/system/test_audit_append_only.py similarity index 98% rename from tests/test_audit_append_only.py rename to tests/system/test_audit_append_only.py index de7e957..34148b8 100644 --- a/tests/test_audit_append_only.py +++ b/tests/system/test_audit_append_only.py @@ -19,7 +19,7 @@ from api.config import settings from api.services.audit import acting_as, verify_chain, write_audit -_ALEMBIC_INI = str(Path(__file__).resolve().parent.parent / "alembic.ini") +_ALEMBIC_INI = str(Path(__file__).resolve().parents[2] / "alembic.ini") def _connect(): diff --git a/tests/test_failed_uploads.py b/tests/system/test_failed_uploads.py similarity index 86% rename from tests/test_failed_uploads.py rename to tests/system/test_failed_uploads.py index 55c358c..cffc221 100644 --- a/tests/test_failed_uploads.py +++ b/tests/system/test_failed_uploads.py @@ -10,31 +10,20 @@ import uuid import pytest -from fastapi.testclient import TestClient from psycopg2 import sql from api.config import settings from api.db import Conn -from api.main import app +# self.client comes from live_api; registry_cleanup deletes what these tests registered. @pytest.mark.integration +@pytest.mark.usefixtures("destructive_enabled", "live_api", "registry_cleanup") class EveryRowFailed(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._orig_allow_destructive = settings.allow_destructive - settings.allow_destructive = True - cls.ctx = TestClient(app) - cls.client = cls.ctx.__enter__() - - @classmethod - def tearDownClass(cls): - settings.allow_destructive = cls._orig_allow_destructive - cls.ctx.__exit__(None, None, None) - def setUp(self): self.tag = f"{os.getpid()}_{uuid.uuid4().hex[:8]}" self.name = f"allfail_{self.tag}.csv" + self.registry_cleanup.append(self.name) def _upload(self, content: str, types: list[str], overwrite: bool = False) -> dict: return self.client.post("/api/csv/upload", json={ diff --git a/tests/test_header_mapping.py b/tests/system/test_header_mapping.py similarity index 100% rename from tests/test_header_mapping.py rename to tests/system/test_header_mapping.py diff --git a/tests/test_import_counts.py b/tests/system/test_import_counts.py similarity index 100% rename from tests/test_import_counts.py rename to tests/system/test_import_counts.py diff --git a/tests/test_issue_04_multi_file_upload.py b/tests/system/test_issue_04_multi_file_upload.py similarity index 100% rename from tests/test_issue_04_multi_file_upload.py rename to tests/system/test_issue_04_multi_file_upload.py diff --git a/tests/test_issue_05_import_summary.py b/tests/system/test_issue_05_import_summary.py similarity index 100% rename from tests/test_issue_05_import_summary.py rename to tests/system/test_issue_05_import_summary.py diff --git a/tests/test_migrations.py b/tests/system/test_migrations.py similarity index 99% rename from tests/test_migrations.py rename to tests/system/test_migrations.py index ad818b4..d372384 100644 --- a/tests/test_migrations.py +++ b/tests/system/test_migrations.py @@ -28,7 +28,7 @@ _HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/" "PGPASSWORD/PGDATABASE). See scripts/start-api.ps1.") -_ALEMBIC_INI = str(Path(__file__).resolve().parent.parent / "alembic.ini") +_ALEMBIC_INI = str(Path(__file__).resolve().parents[2] / "alembic.ini") def _connect(): diff --git a/tests/test_oidc.py b/tests/system/test_oidc.py similarity index 100% rename from tests/test_oidc.py rename to tests/system/test_oidc.py diff --git a/tests/test_original_bytes.py b/tests/system/test_original_bytes.py similarity index 100% rename from tests/test_original_bytes.py rename to tests/system/test_original_bytes.py diff --git a/tests/test_password_reset.py b/tests/system/test_password_reset.py similarity index 98% rename from tests/test_password_reset.py rename to tests/system/test_password_reset.py index 2ca9c14..15c0ef3 100644 --- a/tests/test_password_reset.py +++ b/tests/system/test_password_reset.py @@ -18,10 +18,10 @@ from api.db import Conn from api.services import accounts from api.tenancy import migrate_tenant, schema_for -from tests.test_accounts import _PW, _Accounts +from tests.system.test_accounts import _PW, _Accounts _NEW = "a brand new passphrase" -_ROOT = Path(__file__).resolve().parent.parent +_ROOT = Path(__file__).resolve().parents[2] def _bearer(token: str) -> dict: diff --git a/tests/test_preserved_uploads.py b/tests/system/test_preserved_uploads.py similarity index 100% rename from tests/test_preserved_uploads.py rename to tests/system/test_preserved_uploads.py diff --git a/tests/test_ragged_rows.py b/tests/system/test_ragged_rows.py similarity index 100% rename from tests/test_ragged_rows.py rename to tests/system/test_ragged_rows.py diff --git a/tests/test_registry_race.py b/tests/system/test_registry_race.py similarity index 100% rename from tests/test_registry_race.py rename to tests/system/test_registry_race.py diff --git a/tests/test_review.py b/tests/system/test_review.py similarity index 88% rename from tests/test_review.py rename to tests/system/test_review.py index 1b40730..5d516b6 100644 --- a/tests/test_review.py +++ b/tests/system/test_review.py @@ -109,29 +109,25 @@ def test_refusals(self): self.assertEqual(res.status_code, code) self.assertEqual(self.client.get(f"/api/csv/uploads/{uid}/rows/2/history").json(), []) + @pytest.mark.usefixtures("te_org_cleanup") def test_te_review_shows_the_database_reason_a_row_was_rejected(self): # #190: T&E uploads stored no types, so review said errors: [] for a row the CHECK rejected. good, bad = f"TE_REV_{self.tag}_ok", f"TE_REV_{self.tag}_bad" + self.te_org_cleanup.extend([good, bad]) res = self.client.post("/api/csv/upload", json={ "fileName": f"te_rev_{self.tag}.csv", "mode": "te", "targetTable": "organisations", "content": f"name,org_type,country\n{good},government,AU\n{bad},not_a_type,AU\n"}).json() - try: - self.assertEqual(res["failedRows"], 1, res) - page = self._rows(res["uploadId"]) - self.assertEqual(len(page["types"]), 3) - ok_row, bad_row = page["rows"] - self.assertEqual(ok_row["errors"], []) - [err] = bad_row["errors"] - self.assertEqual(err["column"], "(database)") - self.assertIn("org_type", err["reason"]) - # Once a reviewer edits the row, the upload-time reason no longer describes it. - self.assertEqual(self._edit(res["uploadId"], 2, "org_type", "prime").status_code, 200) - self.assertEqual(self._rows(res["uploadId"])["rows"][1]["errors"], []) - finally: - with Conn() as conn, conn.cursor() as cur: - cur.execute(sql.SQL("DELETE FROM {}.organisations WHERE name = ANY(%s)").format( - sql.Identifier(settings.TE_SCHEMA)), ([good, bad],)) - conn.commit() + self.assertEqual(res["failedRows"], 1, res) + page = self._rows(res["uploadId"]) + self.assertEqual(len(page["types"]), 3) + ok_row, bad_row = page["rows"] + self.assertEqual(ok_row["errors"], []) + [err] = bad_row["errors"] + self.assertEqual(err["column"], "(database)") + self.assertIn("org_type", err["reason"]) + # Once a reviewer edits the row, the upload-time reason no longer describes it. + self.assertEqual(self._edit(res["uploadId"], 2, "org_type", "prime").status_code, 200) + self.assertEqual(self._rows(res["uploadId"])["rows"][1]["errors"], []) def test_changes_are_audited_and_insert_only(self): uid = self._three_rows() diff --git a/tests/test_review_cases.py b/tests/system/test_review_cases.py similarity index 100% rename from tests/test_review_cases.py rename to tests/system/test_review_cases.py diff --git a/tests/test_roles.py b/tests/system/test_roles.py similarity index 98% rename from tests/test_roles.py rename to tests/system/test_roles.py index 55d2775..79c5a6e 100644 --- a/tests/test_roles.py +++ b/tests/system/test_roles.py @@ -24,7 +24,7 @@ from api.tenancy import migrate_tenant, schema_for _PW = "correct horse battery staple" -_DOC = Path(__file__).resolve().parent.parent / "docs" / "ROLES.md" +_DOC = Path(__file__).resolve().parents[2] / "docs" / "ROLES.md" # action -> (method, path); {table} and {id} are filled per run. _ROUTES = { "preview": ("POST", "/api/csv/preview"), diff --git a/tests/test_runtime_role.py b/tests/system/test_runtime_role.py similarity index 99% rename from tests/test_runtime_role.py rename to tests/system/test_runtime_role.py index 0ac0858..8b7c7b3 100644 --- a/tests/test_runtime_role.py +++ b/tests/system/test_runtime_role.py @@ -25,7 +25,7 @@ from api.db import Conn from api.main import app -_ROOT = Path(__file__).resolve().parent.parent +_ROOT = Path(__file__).resolve().parents[2] def _owner(): diff --git a/tests/test_te_loader_integration.py b/tests/system/test_te_loader_integration.py similarity index 100% rename from tests/test_te_loader_integration.py rename to tests/system/test_te_loader_integration.py diff --git a/tests/test_te_reupload_registry.py b/tests/system/test_te_reupload_registry.py similarity index 100% rename from tests/test_te_reupload_registry.py rename to tests/system/test_te_reupload_registry.py diff --git a/tests/test_tenant_isolation.py b/tests/system/test_tenant_isolation.py similarity index 100% rename from tests/test_tenant_isolation.py rename to tests/system/test_tenant_isolation.py diff --git a/tests/system_integration/__init__.py b/tests/system_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/system_integration/environments/__init__.py b/tests/system_integration/environments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/environments/_contract.py b/tests/system_integration/environments/_contract.py similarity index 90% rename from tests/environments/_contract.py rename to tests/system_integration/environments/_contract.py index 7084a6c..1a77418 100644 --- a/tests/environments/_contract.py +++ b/tests/system_integration/environments/_contract.py @@ -1,6 +1,7 @@ """Shared contract every deployed environment must satisfy.""" from __future__ import annotations -import shutil, subprocess # nosec B404 +import shutil +import subprocess # nosec B404 CORE_TABLES = { "organisations", "personnel", "test_programs", "temp_documents", @@ -25,7 +26,12 @@ def _psql(db, sql): class EnvironmentContract: - ENV = ""; DB = ""; SCHEMA = ""; APP_USER = ""; CONN_LIMIT = 0; SEEDED = False + ENV = "" + DB = "" + SCHEMA = "" + APP_USER = "" + CONN_LIMIT = 0 + SEEDED = False @classmethod def setUpClass(cls): @@ -35,7 +41,8 @@ def setUpClass(cls): def test_database_exists(self): ok, out = _psql("postgres", f"SELECT 1 FROM pg_database WHERE datname = '{self.DB}'") - self.assertTrue(ok, out); self.assertEqual(out, "1") + self.assertTrue(ok, out) + self.assertEqual(out, "1") def test_schema_has_all_core_tables(self): ok, out = _psql(self.DB, f"SELECT table_name FROM information_schema.tables WHERE table_schema = '{self.SCHEMA}'") @@ -45,7 +52,8 @@ def test_schema_has_all_core_tables(self): def test_application_role_exists(self): ok, out = _psql(self.DB, f"SELECT 1 FROM pg_roles WHERE rolname = '{self.APP_USER}'") - self.assertTrue(ok, out); self.assertEqual(out, "1") + self.assertTrue(ok, out) + self.assertEqual(out, "1") def test_application_role_connection_limit(self): ok, out = _psql(self.DB, f"SELECT COALESCE(rolconnlimit,-999) FROM pg_roles WHERE rolname = '{self.APP_USER}'") diff --git a/tests/system_integration/environments/test_env_dev.py b/tests/system_integration/environments/test_env_dev.py new file mode 100644 index 0000000..23d64a2 --- /dev/null +++ b/tests/system_integration/environments/test_env_dev.py @@ -0,0 +1,15 @@ +"""Contract tests for the dev environment.""" +import unittest +import pytest +from tests.system_integration.environments._contract import EnvironmentContract + +pytestmark = [pytest.mark.integration, pytest.mark.parity] + + +class DevEnvironment(EnvironmentContract, unittest.TestCase): + ENV = "dev" + DB = "te_mgmt_dev" + SCHEMA = "te_dev" + APP_USER = "te_dev_user" + CONN_LIMIT = 10 + SEEDED = True diff --git a/tests/system_integration/environments/test_env_prod.py b/tests/system_integration/environments/test_env_prod.py new file mode 100644 index 0000000..71412dd --- /dev/null +++ b/tests/system_integration/environments/test_env_prod.py @@ -0,0 +1,15 @@ +"""Contract tests for the prod environment.""" +import unittest +import pytest +from tests.system_integration.environments._contract import EnvironmentContract + +pytestmark = [pytest.mark.integration, pytest.mark.parity] + + +class ProdEnvironment(EnvironmentContract, unittest.TestCase): + ENV = "prod" + DB = "te_mgmt_prod" + SCHEMA = "te_prod" + APP_USER = "te_prod_user" + CONN_LIMIT = 50 + SEEDED = False diff --git a/tests/system_integration/environments/test_env_staging.py b/tests/system_integration/environments/test_env_staging.py new file mode 100644 index 0000000..0d8b57d --- /dev/null +++ b/tests/system_integration/environments/test_env_staging.py @@ -0,0 +1,15 @@ +"""Contract tests for the staging environment.""" +import unittest +import pytest +from tests.system_integration.environments._contract import EnvironmentContract + +pytestmark = [pytest.mark.integration, pytest.mark.parity] + + +class StagingEnvironment(EnvironmentContract, unittest.TestCase): + ENV = "staging" + DB = "te_mgmt_staging" + SCHEMA = "te_staging" + APP_USER = "te_stg_user" + CONN_LIMIT = 25 + SEEDED = False diff --git a/tests/system_integration/environments/test_env_test.py b/tests/system_integration/environments/test_env_test.py new file mode 100644 index 0000000..6e909b1 --- /dev/null +++ b/tests/system_integration/environments/test_env_test.py @@ -0,0 +1,15 @@ +"""Contract tests for the test environment.""" +import unittest +import pytest +from tests.system_integration.environments._contract import EnvironmentContract + +pytestmark = [pytest.mark.integration, pytest.mark.parity] + + +class TestEnvironment(EnvironmentContract, unittest.TestCase): + ENV = "test" + DB = "te_mgmt_test" + SCHEMA = "te_test" + APP_USER = "te_test_user" + CONN_LIMIT = 15 + SEEDED = True diff --git a/tests/test_csv_loader_arbitrary_shapes.py b/tests/system_integration/test_csv_loader_arbitrary_shapes.py similarity index 98% rename from tests/test_csv_loader_arbitrary_shapes.py rename to tests/system_integration/test_csv_loader_arbitrary_shapes.py index d079a36..e3e6c0b 100644 --- a/tests/test_csv_loader_arbitrary_shapes.py +++ b/tests/system_integration/test_csv_loader_arbitrary_shapes.py @@ -21,7 +21,7 @@ pytestmark = pytest.mark.integration -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] LOADER = ROOT / "build" / "csv_loader.sh" UTILISE = ROOT / "build" / "csv_utilise.sh" BUILD_DIR = ROOT / "build" diff --git a/tests/test_e2e_pipeline.py b/tests/system_integration/test_e2e_pipeline.py similarity index 98% rename from tests/test_e2e_pipeline.py rename to tests/system_integration/test_e2e_pipeline.py index 3134bec..d6fd899 100644 --- a/tests/test_e2e_pipeline.py +++ b/tests/system_integration/test_e2e_pipeline.py @@ -19,7 +19,7 @@ pytestmark = pytest.mark.e2e -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] VALIDATOR = ROOT / "build" / "csv" / "validator.py" RUNNER = ROOT / "evals" / "runner.py" @@ -83,7 +83,7 @@ def test_happy_path_two_valid_rows(self): self.assertIn("2,Bob", valid_lines) # skip.csv should contain only the header (no skips) if skip_csv.exists(): - skip_lines = [l for l in skip_csv.read_text(encoding="utf-8").splitlines() if l.strip()] + skip_lines = [ln for ln in skip_csv.read_text(encoding="utf-8").splitlines() if ln.strip()] self.assertLessEqual(len(skip_lines), 1) def test_mixed_rows_splits_correctly(self): diff --git a/tests/test_parity.py b/tests/system_integration/test_parity.py similarity index 99% rename from tests/test_parity.py rename to tests/system_integration/test_parity.py index dfe5943..55d7349 100644 --- a/tests/test_parity.py +++ b/tests/system_integration/test_parity.py @@ -17,7 +17,7 @@ pytestmark = [pytest.mark.integration, pytest.mark.parity] -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] # Expected minimum seed row counts for the dev environment (from seed data). # Adjust these values if the seed SQL is intentionally changed. diff --git a/tests/uat/__init__.py b/tests/uat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_api_hardening.py b/tests/unit/test_api_hardening.py similarity index 100% rename from tests/test_api_hardening.py rename to tests/unit/test_api_hardening.py diff --git a/tests/test_api_unit.py b/tests/unit/test_api_unit.py similarity index 99% rename from tests/test_api_unit.py rename to tests/unit/test_api_unit.py index b217142..ca33159 100644 --- a/tests/test_api_unit.py +++ b/tests/unit/test_api_unit.py @@ -154,7 +154,7 @@ def test_identifiers_are_never_string_formatted(self): import ast import pathlib - root = pathlib.Path(__file__).resolve().parents[1] / "api" + root = pathlib.Path(__file__).resolve().parents[2] / "api" offenders = [] for path in root.rglob("*.py"): tree = ast.parse(path.read_text(encoding="utf-8")) diff --git a/tests/test_ci_quality_gate.py b/tests/unit/test_ci_quality_gate.py similarity index 99% rename from tests/test_ci_quality_gate.py rename to tests/unit/test_ci_quality_gate.py index 2991b2e..7d9df96 100644 --- a/tests/test_ci_quality_gate.py +++ b/tests/unit/test_ci_quality_gate.py @@ -12,7 +12,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.regression] -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".github" / "workflows" / "quality-gate.yml" GAP_ANALYSIS = ROOT / "docs/GAP_ANALYSIS.md" @@ -48,7 +48,6 @@ def test_triggers_on_push_to_main(self): self.assertIn("main", push_branches) - class TestAllJobsHaveConsistentStructure(unittest.TestCase): """All jobs must have checkout, python setup, and artifact upload.""" @@ -94,7 +93,6 @@ def test_all_jobs_install_dev_deps(self): ) - class TestGapAnalysisG2Accepted(unittest.TestCase): """G2 is accepted as out-of-scope, not closed. Checks the decision is recorded in docs/GAP_ANALYSIS.md. diff --git a/tests/test_csv_utilise.py b/tests/unit/test_csv_utilise.py similarity index 98% rename from tests/test_csv_utilise.py rename to tests/unit/test_csv_utilise.py index 834e23d..7aeee41 100644 --- a/tests/test_csv_utilise.py +++ b/tests/unit/test_csv_utilise.py @@ -15,7 +15,7 @@ pytestmark = pytest.mark.unit -SCRIPT = Path(__file__).resolve().parents[1] / "build" / "csv_utilise.sh" +SCRIPT = Path(__file__).resolve().parents[2] / "build" / "csv_utilise.sh" def _find_bash(): diff --git a/tests/test_csv_validator.py b/tests/unit/test_csv_validator.py similarity index 98% rename from tests/test_csv_validator.py rename to tests/unit/test_csv_validator.py index f8ad4a3..c26fee2 100644 --- a/tests/test_csv_validator.py +++ b/tests/unit/test_csv_validator.py @@ -18,7 +18,7 @@ class CsvValidatorTests(unittest.TestCase): def setUp(self): - self.validator = Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py" + self.validator = Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py" def run_validator(self, env): result = subprocess.run( diff --git a/tests/test_evals_runner.py b/tests/unit/test_evals_runner.py similarity index 98% rename from tests/test_evals_runner.py rename to tests/unit/test_evals_runner.py index 8ed9840..b37aa8c 100644 --- a/tests/test_evals_runner.py +++ b/tests/unit/test_evals_runner.py @@ -17,7 +17,7 @@ pytestmark = pytest.mark.unit -PROJECT_ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = Path(__file__).resolve().parents[2] RUNNER_PATH = PROJECT_ROOT / "evals" / "runner.py" spec = importlib.util.spec_from_file_location("evals_runner", RUNNER_PATH) diff --git a/tests/test_evals_tier_e_x.py b/tests/unit/test_evals_tier_e_x.py similarity index 97% rename from tests/test_evals_tier_e_x.py rename to tests/unit/test_evals_tier_e_x.py index 51efa99..f63d45a 100644 --- a/tests/test_evals_tier_e_x.py +++ b/tests/unit/test_evals_tier_e_x.py @@ -17,7 +17,7 @@ import pytest -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "evals")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "evals")) import runner # noqa: E402 @@ -168,7 +168,7 @@ def test_sqlite_internal_tables_are_excluded(self): def test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file(self): import json - root = Path(__file__).resolve().parent.parent + root = Path(__file__).resolve().parents[2] ddl = (root / "build" / "schema" / "sqlite" / "te_core_schema.sql").read_text(encoding="utf-8") exp = json.loads( @@ -185,7 +185,7 @@ def test_known_missing_tables_are_genuinely_absent(self): # these tables to the DDL but forgets to move it out of # known_missing_tables, the file would misreport the shortfall. import json - root = Path(__file__).resolve().parent.parent + root = Path(__file__).resolve().parents[2] ddl = (root / "build" / "schema" / "sqlite" / "te_core_schema.sql").read_text(encoding="utf-8") exp = json.loads( @@ -202,7 +202,7 @@ def test_known_missing_tables_are_genuinely_absent(self): def test_missing_plus_present_accounts_for_all_twelve_tables(self): import json - root = Path(__file__).resolve().parent.parent + root = Path(__file__).resolve().parents[2] exp = json.loads( (root / "evals" / "expected" / "tier_x" / "01_sqlite_schema_structure.json").read_text(encoding="utf-8") diff --git a/tests/test_preflight_ps1.py b/tests/unit/test_preflight_ps1.py similarity index 98% rename from tests/test_preflight_ps1.py rename to tests/unit/test_preflight_ps1.py index b6e3619..e35d1bd 100644 --- a/tests/test_preflight_ps1.py +++ b/tests/unit/test_preflight_ps1.py @@ -19,7 +19,7 @@ import pytest -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "preflight.ps1" WINDOWS_PS = shutil.which("powershell") if sys.platform == "win32" else None @@ -201,4 +201,4 @@ def test_port_label_follows_pgport(): def test_pg_checks_are_optional_never_fail(): r = _run([], extra_env={"PGHOST": "localhost", "PGPORT": "5999", "PGPASSWORD": "x"}) pg = r.stdout.split("PostgreSQL")[-1].split("Summary:")[0] - assert "FAIL" not in pg, r.stdout + r.stderr \ No newline at end of file + assert "FAIL" not in pg, r.stdout + r.stderr diff --git a/tests/test_regression.py b/tests/unit/test_regression.py similarity index 99% rename from tests/test_regression.py rename to tests/unit/test_regression.py index 536772f..814a0c1 100644 --- a/tests/test_regression.py +++ b/tests/unit/test_regression.py @@ -15,7 +15,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.regression] -VALIDATOR = Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py" +VALIDATOR = Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py" def _run(env: dict) -> subprocess.CompletedProcess: diff --git a/tests/test_security.py b/tests/unit/test_security.py similarity index 99% rename from tests/test_security.py rename to tests/unit/test_security.py index cbd8cb6..286c2bd 100644 --- a/tests/test_security.py +++ b/tests/unit/test_security.py @@ -14,7 +14,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.security] -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] # ── Python source files to scan ─────────────────────────────────────────────── PY_SOURCE = [ diff --git a/tests/test_service_loader.py b/tests/unit/test_service_loader.py similarity index 100% rename from tests/test_service_loader.py rename to tests/unit/test_service_loader.py diff --git a/tests/test_snapshot.py b/tests/unit/test_snapshot.py similarity index 95% rename from tests/test_snapshot.py rename to tests/unit/test_snapshot.py index 805b551..cf1b087 100644 --- a/tests/test_snapshot.py +++ b/tests/unit/test_snapshot.py @@ -18,8 +18,8 @@ pytestmark = [pytest.mark.unit, pytest.mark.snapshot] -VALIDATOR = Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py" -SNAPSHOTS_DIR = Path(__file__).resolve().parent / "snapshots" +VALIDATOR = Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py" +SNAPSHOTS_DIR = Path(__file__).resolve().parents[1] / "snapshots" UPDATE = os.environ.get("UPDATE_SNAPSHOTS", "").lower() in ("1", "true", "yes") @@ -95,7 +95,7 @@ def test_all_valid_no_skips(self): self.assertEqual(exit_code, 0) _assert_snapshot(valid_text, "all_valid_expected_valid.csv") # skip file should have only a header (or be empty) - skip_lines = [l for l in skip_text.splitlines() if l.strip()] + skip_lines = [ln for ln in skip_text.splitlines() if ln.strip()] self.assertLessEqual(len(skip_lines), 1, "Expected at most a header row in skip.csv when all rows are valid")