diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 4bfab3c..cbf74a8 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -164,17 +164,22 @@ jobs: # the build on any skip. # The API reads PGDATABASE (api/config.py). The schema lives in # te_mgmt_dev, not the job-level default "postgres" (issue #45). + # Accessibility tests need Playwright browsers and a running frontend, + # neither of which exists in this job — they run in the `accessibility` + # job below. Deselected, not skipped: test_report.py accounts for them + # under NOT RUN. - name: Full test suite — final result with skip accounting env: PGDATABASE: te_mgmt_dev - run: python3 scripts/test_report.py --strict + run: python3 scripts/test_report.py --markers "not accessibility" --strict # #194: measure api/services coverage with per-loader floors. + # Accessibility tests need a browser and a running frontend (own job below). - name: api/services coverage gate env: PGDATABASE: te_mgmt_dev run: | - python -m pytest tests/ \ + python -m pytest tests/ -m "not accessibility" \ --cov=api/services \ --cov-report=json:coverage-services.json \ --cov-report=term-missing \ @@ -189,3 +194,82 @@ jobs: path: evals/reports/ if-no-files-found: ignore retention-days: 30 + + # WCAG 2.1 AA scans (#168). Needs a browser and a running frontend, so it + # cannot ride either job above. + # + # The scan step is continue-on-error ON PURPOSE: the frontend has known, + # unfixed accessibility defects (sonner toast contrast, selects without + # accessible names, download-menu aria-labels, upload progress bar), so the + # scans fail today and that failure is the backlog, not a regression. The + # report is uploaded as an artifact either way. Remove continue-on-error — + # and only then add this job to branch protection — once the frontend fixes + # land. + accessibility: + name: accessibility (WCAG 2.1 AA) + runs-on: ubuntu-latest + if: github.event_name == 'pull_request' + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + requirements-dev.txt + api/requirements.txt + + - name: Install dev dependencies + run: pip install -r requirements-dev.txt -r api/requirements.txt + + - name: Install Playwright Chromium + run: python -m playwright install --with-deps chromium + + - name: Setup Node + uses: actions/setup-node@v5 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: csv-table-hub-main/package-lock.json + + # npm, not bun: bun.lock is out of step with package.json (zod ^3 vs ^4), + # so `bun install --frozen-lockfile` fails; package-lock.json is current. + - name: Install frontend dependencies + working-directory: csv-table-hub-main + run: npm ci --no-audit --no-fund + + - name: Start the Vite dev server + working-directory: csv-table-hub-main + run: | + npx vite dev --host 127.0.0.1 --port 5173 --strictPort > "$RUNNER_TEMP/vite.log" 2>&1 & + for i in $(seq 1 60); do + if curl -fsS -o /dev/null http://127.0.0.1:5173/auth; then + echo "dev server up after ${i}s"; exit 0 + fi + sleep 1 + done + echo "::error::dev server did not answer on 127.0.0.1:5173 within 60s" + cat "$RUNNER_TEMP/vite.log" + exit 1 + + - name: WCAG 2.1 AA scans + continue-on-error: true + env: + A11Y_BASE_URL: http://127.0.0.1:5173 + run: | + python -m pytest tests/accessibility -v \ + --junitxml="$RUNNER_TEMP/accessibility.xml" + + - name: Upload accessibility results + if: always() + uses: actions/upload-artifact@v7 + with: + name: accessibility-results + path: | + ${{ runner.temp }}/accessibility.xml + ${{ runner.temp }}/vite.log + if-no-files-found: ignore + retention-days: 30 diff --git a/.gitignore b/.gitignore index 2af18ef..7a0ccbc 100644 --- a/.gitignore +++ b/.gitignore @@ -46,9 +46,9 @@ _norton_/ extend to Oracle dbs/ # Runtime test artifacts (regenerate with scripts/test_report.py) -# tests/snapshots/ holds committed fixtures AND runtime-generated output. +# tests/unit/snapshots/ holds committed fixtures AND runtime-generated output. # Ignore only the generated file; fixtures must be committed. -tests/snapshots/all_valid_expected_valid.csv +tests/unit/snapshots/all_valid_expected_valid.csv tfplan *.tfplan terraform-provider-*.log diff --git a/Makefile b/Makefile index f256b58..4936ca7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,6 @@ .PHONY: test-free test-gate test-evals test-e2e test-all \ + test-unit test-system test-sit test-performance test-accessibility test-uat \ + test-catalogue \ lint lint-diff health \ eval-list eval-compare eval-summary select-tests \ csv-load csv-list csv-demo @@ -32,6 +34,34 @@ test-all: pytest tests/ --cov=build/csv --cov=evals --cov-report=term-missing python3 evals/runner.py --tiers p,i,s,e --verbose +# ── Test categories (#167) ──────────────────────────────────────────────────── +# One target per folder under tests/. tests/conftest.py gives every test the +# marker of its folder, so `make test-sit` and `pytest -m sit` select the same +# set. unit needs nothing; the rest need PostgreSQL, and accessibility also +# needs Playwright browsers plus a running frontend (see tests/README.md). + +test-unit: + pytest tests/unit -v + +test-system: + pytest tests/system -v + +test-sit: + pytest tests/system_integration -v + +test-performance: + pytest tests/performance -v + +test-accessibility: + pytest tests/accessibility -v + +test-uat: + pytest tests/uat -v + +# Regenerate docs/TEST_CASES.md from the test tree (#169). +test-catalogue: + python3 scripts/test_catalogue.py + # ── Code quality ────────────────────────────────────────────────────────────── # Full flake8 + bandit scan over all Python source. diff --git a/README.md b/README.md index 8cf1d6a..4656ce7 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ organisations ──< personnel | Connection Limit | 10 | 15 | 25 | 50 | | Seed Data | ✅ | ✅ | ❌ | ❌ | -Staging and Prod are **never seeded**. The `tests/environments/` contract tests +Staging and Prod are **never seeded**. The `tests/system_integration/environments/` contract tests verify this automatically. --- diff --git a/api/auth.py b/api/auth.py index 93c920a..e814069 100644 --- a/api/auth.py +++ b/api/auth.py @@ -78,7 +78,7 @@ async def tenant_scope(authorization: str = Header(default=""), # M2c (#94): which roles may perform each action. docs/ROLES.md shows the same -# table, and tests/test_roles.py checks that the two agree. +# table, and tests/system_integration/test_roles.py checks that the two agree. PERMISSIONS: dict[str, frozenset[str]] = { "preview": frozenset({"uploader", "tenant_admin"}), "upload": frozenset({"uploader", "tenant_admin"}), diff --git a/api/routers/audit_routes.py b/api/routers/audit_routes.py index 49e7d6c..85cb347 100644 --- a/api/routers/audit_routes.py +++ b/api/routers/audit_routes.py @@ -5,7 +5,7 @@ O-04 update: the table is no longer delete-only. Successful uploads also write a row, through the shared writer in api/services/audit.py, so this feed now covers creations as well as destructions. Integration coverage lives in -tests/test_api_integration.py::AuditLogTrail.""" +tests/system_integration/test_api_integration.py::AuditLogTrail.""" from __future__ import annotations diff --git a/api/services/dynamic_loader.py b/api/services/dynamic_loader.py index b8464d2..448bed2 100644 --- a/api/services/dynamic_loader.py +++ b/api/services/dynamic_loader.py @@ -125,19 +125,16 @@ def upload_dynamic( try: return _do_upload(file_name, types, overwrite, logs, schema, rows, file_hash, session=session) - except psycopg2.errors.UniqueViolation: + except (psycopg2.errors.UniqueViolation, psycopg2.errors.DuplicateTable, + psycopg2.errors.DuplicateObject): # #81: an identical concurrent upload won a race that _do_upload could # not settle itself (e.g. both creating csv_ at once). Nothing - # from this request was committed. + # from this request was committed. Which error the loser sees depends on + # where the race lands: a unique index (UniqueViolation), the table + # (DuplicateTable) or its row type (DuplicateObject) — #191 caught the + # last two surfacing as a generic "database error". _log(logs, "duplicate_check", "Lost race to a concurrent identical upload", "warn") - return { - "status": "duplicate_file", - "reason": "content", - "existingFileName": file_name, - "tableName": f"csv_{file_hash[:16]}", - "existingRowCount": 0, - "logs": logs, - } + return _lost_race(logs, schema, file_name, file_hash) except psycopg2.Error as exc: # Mirror te_loader's structured-error contract: an unexpected DB error # (e.g. a column type edge case not caught by cast_value) should not @@ -151,6 +148,25 @@ def upload_dynamic( } +def _lost_race(logs: list[dict], schema: str, file_name: str, file_hash: str) -> dict: + """Name the upload that won. The winner commits its table and its registry + row together, so once the loser has seen the clash the row is visible.""" + with Conn() as conn, conn.cursor() as cur: + cur.execute( + sql.SQL("SELECT file_name, table_name, row_count, file_hash FROM {}.csv_files " + "WHERE file_hash = %s OR file_name = %s ORDER BY (file_hash = %s) DESC LIMIT 1") + .format(sql.Identifier(schema)), (file_hash, file_name, file_hash)) + existing = cur.fetchone() + return { + "status": "duplicate_file", + "reason": "content" if existing and existing[3] == file_hash else "name", + "existingFileName": existing[0] if existing else file_name, + "tableName": existing[1] if existing else f"csv_{file_hash[:16]}", + "existingRowCount": (existing[2] or 0) if existing else 0, + "logs": logs, + } + + def _do_upload( file_name: str, types: list[str] | None, diff --git a/csv-table-hub-main/src/__tests__/export.test.ts b/csv-table-hub-main/src/__tests__/export.test.ts index 2f93ad5..37fdb56 100644 --- a/csv-table-hub-main/src/__tests__/export.test.ts +++ b/csv-table-hub-main/src/__tests__/export.test.ts @@ -305,3 +305,56 @@ describe("downloadAuditReport", () => { expect(text).toContain("Imported"); }); }); +// ─── XLSX formula guard — the cell is stored as text (#193) ─────────────────── +// The tests above read values back through sheet_to_json. What actually stops a +// spreadsheet evaluating a cell is that it is stored as TEXT with no formula, so +// assert that directly, for each payload class the issue names, and for the audit +// report as well as the error report. + +describe("XLSX formula guard: cell type", () => { + let cap: ReturnType; + + beforeEach(() => { + cap = captureBlob(); + }); + afterEach(() => { + cap.reset(); + }); + + async function cellAt(addr: string, sheet: string): Promise { + const wb = XLSX.read(await cap.blob!.arrayBuffer(), { type: "array" }); + return wb.Sheets[sheet][addr]; + } + + it.each(["=cmd", "+cmd", "-cmd", "@cmd", "\t=cmd"])( + "error report: %j is a text cell with no formula, carrying the guard prefix", + async (value) => { + const rows = buildErrorRows("d.csv", [{ rowNumber: 1, column: "c", value, reason: "bad" }]); + downloadErrorReport("d.csv", rows, "xlsx"); + const cell = await cellAt("D2", "Errors"); // ERROR_HEADERS: file,row_number,column,value,... + expect(cell.t).toBe("s"); + expect(cell.f).toBeUndefined(); + expect(cell.v).toBe(`'${value}`); + }, + ); + + it("audit report: a formula-like file name is guarded too", async () => { + downloadAuditReport( + [ + { + file: "=cmd", + status: "Imported", + imported_at: "x", + total_rows: 1, + inserted_rows: 1, + duplicate_rows_skipped: 0, + failed_rows: 0, + table_name: "t", + }, + ], + "xlsx", + ); + const cell = await cellAt("A2", "Audit"); + expect([cell.t, cell.f, cell.v]).toEqual(["s", undefined, "'=cmd"]); + }); +}); diff --git a/docs/TEST_CASES.md b/docs/TEST_CASES.md index 9dd0768..ab129ec 100644 --- a/docs/TEST_CASES.md +++ b/docs/TEST_CASES.md @@ -1,79 +1,21 @@ # Test Catalogue > Auto-generated by `scripts/test_catalogue.py` — do not edit by hand. -> 732 tests across 38 files. +> 738 tests across 39 files. To regenerate: `python3 scripts/test_catalogue.py` -## `tests/test_accounts.py` +## `tests/accessibility/test_accessibility.py` | Test | Markers | Objective | |---|---|---| -| `Passwords.test_round_trip` | unit | — | - -| `Passwords.test_wrong_password_fails` | unit | — | - -| `Passwords.test_same_password_hashes_differently` | unit | — | - -| `Passwords.test_malformed_hash_fails_closed` | unit | — | - -| `SignIn.test_login_returns_a_session_for_the_tenant_and_role` | integration | — | - -| `SignIn.test_every_login_failure_is_the_same_401` | integration | — | - -| `SignIn.test_inactive_user_cannot_sign_in_and_loses_live_sessions` | integration | — | - -| `SignIn.test_logout_revokes_the_session` | integration | — | - -| `SignIn.test_expired_session_is_rejected` | integration | — | - -| `SignIn.test_removing_the_membership_ends_the_session` | integration | — | - -| `SignIn.test_missing_or_malformed_token_is_401` | integration | — | - -| `SignIn.test_tokens_are_stored_only_as_hashes` | integration | — | - -| `Lockout.test_eleventh_attempt_is_refused_even_with_the_right_password` | integration | — | - -| `Lockout.test_one_short_of_the_limit_still_signs_in_and_resets_the_count` | integration | — | - -| `Lockout.test_sign_in_works_again_after_the_lock_expires` | integration | — | - -| `Lockout.test_failures_outside_the_window_start_a_new_count` | integration | — | - -| `test_round_trip` | — | — | - -| `test_wrong_password_fails` | — | — | - -| `test_same_password_hashes_differently` | — | — | - -| `test_malformed_hash_fails_closed` | — | — | - -| `test_login_returns_a_session_for_the_tenant_and_role` | — | — | - -| `test_every_login_failure_is_the_same_401` | — | — | - -| `test_inactive_user_cannot_sign_in_and_loses_live_sessions` | — | — | - -| `test_logout_revokes_the_session` | — | — | - -| `test_expired_session_is_rejected` | — | — | - -| `test_removing_the_membership_ends_the_session` | — | — | - -| `test_missing_or_malformed_token_is_401` | — | — | - -| `test_tokens_are_stored_only_as_hashes` | — | — | - -| `test_eleventh_attempt_is_refused_even_with_the_right_password` | — | — | - -| `test_one_short_of_the_limit_still_signs_in_and_resets_the_count` | — | — | +| `test_page_has_no_wcag_21_aa_violations` | — | — | -| `test_sign_in_works_again_after_the_lock_expires` | — | — | +| `test_page_has_one_main_landmark` | — | Landmarks are how a screen-reader user skips the page chrome. | -| `test_failures_outside_the_window_start_a_new_count` | — | — | +| `test_sign_in_form_fields_all_have_labels` | — | A form you cannot label is a form a screen-reader user cannot fill in. | -## `tests/test_api_coverage.py` +## `tests/system/test_api_coverage.py` | Test | Markers | Objective | |---|---|---| @@ -233,107 +175,119 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_delete_blocked_returns_403` | — | — | -## `tests/test_api_hardening.py` +## `tests/system/test_csv_loader_arbitrary_shapes.py` | Test | Markers | Objective | |---|---|---| -| `AuthFailsClosed.test_dev_may_run_without_a_key` | unit | — | +| `CsvLoaderArbitraryShapes.test_shape_tiny` | — | — | -| `AuthFailsClosed.test_any_key_is_accepted_in_dev` | unit | — | +| `CsvLoaderArbitraryShapes.test_shape_medium` | — | — | -| `AuthFailsClosed.test_prod_without_a_key_refuses_to_start` | unit | — | +| `CsvLoaderArbitraryShapes.test_shape_skinny` | — | — | -| `AuthFailsClosed.test_staging_and_test_also_refuse` | unit | — | +| `test_shape_tiny` | — | — | -| `AuthFailsClosed.test_deployed_env_with_a_key_starts` | unit | — | +| `test_shape_medium` | — | — | -| `RateLimiterAlgorithm.test_requests_under_budget_are_allowed` | unit | — | +| `test_shape_skinny` | — | — | -| `RateLimiterAlgorithm.test_request_over_budget_is_refused` | unit | — | +## `tests/system/test_e2e_pipeline.py` -| `RateLimiterAlgorithm.test_retry_after_points_at_the_oldest_hit_leaving_the_window` | unit | — | +| Test | Markers | Objective | +|---|---|---| +| `E2EPipelineValidateOnly.test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | -| `RateLimiterAlgorithm.test_window_expiry_frees_the_budget` | unit | — | +| `E2EPipelineValidateOnly.test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | -| `RateLimiterAlgorithm.test_zero_budget_disables_the_limit` | unit | — | +| `E2EPipelineValidateOnly.test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | -| `RateLimiterAlgorithm.test_read_and_write_scopes_are_independent` | unit | — | +| `E2EPipelineValidateOnly.test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | -| `RateLimiterAlgorithm.test_clients_are_independent` | unit | — | +| `E2EPipelineWithDatabase.test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | -| `RateLimiterAlgorithm.test_bucket_table_is_bounded` | unit | — | +| `E2EPipelineWithDatabase.test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | -| `RateLimiterAlgorithm.test_reset_clears_state` | unit | — | +| `test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | -| `RateLimitMiddleware.test_exceeding_the_write_budget_returns_429_with_retry_after` | unit | — | +| `test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | -| `RateLimitMiddleware.test_health_is_never_rate_limited` | unit | — | +| `test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | -| `RateLimitMiddleware.test_openapi_is_never_rate_limited` | unit | — | +| `test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | -| `RateLimitMiddleware.test_disabled_budget_allows_everything` | unit | — | +| `test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | -| `RateLimitMiddleware.test_forwarded_for_cannot_be_used_to_evade_the_limit` | unit | — | +| `test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | -| `AuditWriterContract.test_only_known_actions_are_accepted` | unit | — | +## `tests/system_integration/test_accounts.py` -| `AuditWriterContract.test_unknown_action_raises_before_any_sql_runs` | unit | — | +| Test | Markers | Objective | +|---|---|---| +| `Passwords.test_round_trip` | unit | — | -| `AuditWriterContract.test_valid_action_reaches_the_cursor_with_bound_parameters` | unit | — | +| `Passwords.test_wrong_password_fails` | unit | — | -| `AuditWriterContract.test_file_id_is_coerced_from_the_loaders_string_form` | unit | — | +| `Passwords.test_same_password_hashes_differently` | unit | — | -| `AuditWriterContract.test_unparseable_file_id_degrades_to_null_rather_than_raising` | unit | — | +| `Passwords.test_malformed_hash_fails_closed` | unit | — | -| `test_dev_may_run_without_a_key` | — | — | +| `SignIn.test_login_returns_a_session_for_the_tenant_and_role` | integration | — | -| `test_any_key_is_accepted_in_dev` | — | — | +| `SignIn.test_every_login_failure_is_the_same_401` | integration | — | -| `test_prod_without_a_key_refuses_to_start` | — | — | +| `SignIn.test_inactive_user_cannot_sign_in_and_loses_live_sessions` | integration | — | -| `test_staging_and_test_also_refuse` | — | — | +| `SignIn.test_logout_revokes_the_session` | integration | — | -| `test_deployed_env_with_a_key_starts` | — | — | +| `SignIn.test_expired_session_is_rejected` | integration | — | -| `test_requests_under_budget_are_allowed` | — | — | +| `SignIn.test_removing_the_membership_ends_the_session` | integration | — | -| `test_request_over_budget_is_refused` | — | — | +| `SignIn.test_missing_or_malformed_token_is_401` | integration | — | -| `test_retry_after_points_at_the_oldest_hit_leaving_the_window` | — | — | +| `SignIn.test_tokens_are_stored_only_as_hashes` | integration | — | -| `test_window_expiry_frees_the_budget` | — | — | +| `Lockout.test_eleventh_attempt_is_refused_even_with_the_right_password` | integration | — | -| `test_zero_budget_disables_the_limit` | — | — | +| `Lockout.test_one_short_of_the_limit_still_signs_in_and_resets_the_count` | integration | — | -| `test_read_and_write_scopes_are_independent` | — | — | +| `Lockout.test_sign_in_works_again_after_the_lock_expires` | integration | — | -| `test_clients_are_independent` | — | — | +| `Lockout.test_failures_outside_the_window_start_a_new_count` | integration | — | -| `test_bucket_table_is_bounded` | — | — | +| `test_round_trip` | — | — | -| `test_reset_clears_state` | — | — | +| `test_wrong_password_fails` | — | — | -| `test_exceeding_the_write_budget_returns_429_with_retry_after` | — | — | +| `test_same_password_hashes_differently` | — | — | -| `test_health_is_never_rate_limited` | — | — | +| `test_malformed_hash_fails_closed` | — | — | -| `test_openapi_is_never_rate_limited` | — | — | +| `test_login_returns_a_session_for_the_tenant_and_role` | — | — | -| `test_disabled_budget_allows_everything` | — | — | +| `test_every_login_failure_is_the_same_401` | — | — | -| `test_forwarded_for_cannot_be_used_to_evade_the_limit` | — | — | +| `test_inactive_user_cannot_sign_in_and_loses_live_sessions` | — | — | -| `test_only_known_actions_are_accepted` | — | — | +| `test_logout_revokes_the_session` | — | — | -| `test_unknown_action_raises_before_any_sql_runs` | — | — | +| `test_expired_session_is_rejected` | — | — | -| `test_valid_action_reaches_the_cursor_with_bound_parameters` | — | — | +| `test_removing_the_membership_ends_the_session` | — | — | -| `test_file_id_is_coerced_from_the_loaders_string_form` | — | — | +| `test_missing_or_malformed_token_is_401` | — | — | -| `test_unparseable_file_id_degrades_to_null_rather_than_raising` | — | — | +| `test_tokens_are_stored_only_as_hashes` | — | — | + +| `test_eleventh_attempt_is_refused_even_with_the_right_password` | — | — | + +| `test_one_short_of_the_limit_still_signs_in_and_resets_the_count` | — | — | + +| `test_sign_in_works_again_after_the_lock_expires` | — | — | + +| `test_failures_outside_the_window_start_a_new_count` | — | — | -## `tests/test_api_integration.py` +## `tests/system_integration/test_api_integration.py` | Test | Markers | Objective | |---|---|---| @@ -445,59 +399,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_exhausted_pool_raises_within_timeout` | — | psycopg2's ThreadedConnectionPool.getconn() raises PoolError | -## `tests/test_api_unit.py` - -| Test | Markers | Objective | -|---|---|---| -| `UnitCsvParse.test_sanitize_columns_no_collision_on_stem_conflict` | unit | BUG-036 regression: ["foo","foo","foo_2"] must produce three | - -| `UnitCsvParse.test_sanitize_columns_dedupes_true_duplicates` | unit | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | - -| `UnitCsvParse.test_sanitize_columns_reserved_stems_get_suffix` | unit | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | - -| `RequestValidation.test_preview_rejects_empty_filename` | unit | — | - -| `RequestValidation.test_preview_rejects_empty_content` | unit | — | - -| `RequestValidation.test_preview_rejects_overlong_filename` | unit | — | - -| `RequestValidation.test_upload_rejects_unknown_mode` | unit | — | - -| `RequestValidation.test_upload_te_mode_requires_target_table` | unit | — | - -| `RequestValidation.test_oversized_upload_is_rejected` | unit | — | - -| `TableNameGuards.test_non_upload_tables_are_rejected` | security, unit | — | - -| `TableNameGuards.test_overlong_names_are_rejected` | security, unit | — | - -| `TableNameGuards.test_identifiers_are_never_string_formatted` | security, unit | Every SQL identifier must go through psycopg2.sql.Identifier. | - -| `test_sanitize_columns_no_collision_on_stem_conflict` | — | BUG-036 regression: ["foo","foo","foo_2"] must produce three | - -| `test_sanitize_columns_dedupes_true_duplicates` | — | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | - -| `test_sanitize_columns_reserved_stems_get_suffix` | — | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | - -| `test_preview_rejects_empty_filename` | — | — | - -| `test_preview_rejects_empty_content` | — | — | - -| `test_preview_rejects_overlong_filename` | — | — | - -| `test_upload_rejects_unknown_mode` | — | — | - -| `test_upload_te_mode_requires_target_table` | — | — | - -| `test_oversized_upload_is_rejected` | — | — | - -| `test_non_upload_tables_are_rejected` | — | — | - -| `test_overlong_names_are_rejected` | — | — | - -| `test_identifiers_are_never_string_formatted` | — | Every SQL identifier must go through psycopg2.sql.Identifier. | - -## `tests/test_audit_append_only.py` +## `tests/system_integration/test_audit_append_only.py` | Test | Markers | Objective | |---|---|---| @@ -529,371 +431,629 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_downgrade_removes_the_triggers_and_columns` | — | — | -## `tests/test_ci_quality_gate.py` +## `tests/system_integration/test_failed_uploads.py` | Test | Markers | Objective | |---|---|---| -| `TestQualityGateStructure.test_has_free_tier_job` | — | — | +| `EveryRowFailed.test_every_row_failing_is_an_error_and_saves_nothing` | integration | — | -| `TestQualityGateStructure.test_has_integration_postgres_job` | — | — | +| `EveryRowFailed.test_failed_overwrite_keeps_the_previous_upload` | integration | — | -| `TestQualityGateStructure.test_triggers_on_pull_request` | — | — | +| `EveryRowFailed.test_partly_failed_upload_still_imports_and_reports_the_failures` | integration | — | -| `TestQualityGateStructure.test_triggers_on_push_to_main` | — | — | +| `test_every_row_failing_is_an_error_and_saves_nothing` | — | — | -| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_checkout` | — | — | +| `test_failed_overwrite_keeps_the_previous_upload` | — | — | -| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_python_setup` | — | — | +| `test_partly_failed_upload_still_imports_and_reports_the_failures` | — | — | -| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_artifact_upload` | — | — | +## `tests/system_integration/test_header_mapping.py` -| `TestAllJobsHaveConsistentStructure.test_all_jobs_install_dev_deps` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `ApplyHeaderMapping.test_no_mapping_returns_headers_unchanged` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_row_exists` | — | — | +| `ApplyHeaderMapping.test_rename_replaces_only_the_named_header` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_is_accepted_or_deferred` | — | — | +| `ApplyHeaderMapping.test_blank_rename_keeps_the_original` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_section_exists` | — | — | +| `ApplyHeaderMapping.test_unknown_original_is_rejected` | unit | — | -| `TestGapAnalysisG2Accepted.test_g2_is_not_marked_closed` | — | — | +| `ApplyHeaderMapping.test_original_listed_twice_is_rejected` | unit | — | -| `test_has_free_tier_job` | — | — | +| `ApplyHeaderMapping.test_rename_colliding_with_an_untouched_column_is_rejected` | unit | — | -| `test_has_integration_postgres_job` | — | — | +| `ApplyHeaderMapping.test_collision_is_judged_after_sanitising` | unit | — | -| `test_triggers_on_pull_request` | — | — | +| `ApplyHeaderMapping.test_swapping_two_names_is_allowed` | unit | — | -| `test_triggers_on_push_to_main` | — | — | +| `HeaderMappingOverTheApi.test_renamed_header_becomes_the_database_column` | integration | — | -| `test_all_jobs_have_checkout` | — | — | +| `HeaderMappingOverTheApi.test_upload_without_a_mapping_keeps_csv_headers` | integration | — | -| `test_all_jobs_have_python_setup` | — | — | +| `HeaderMappingOverTheApi.test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | integration | — | -| `test_all_jobs_have_artifact_upload` | — | — | +| `HeaderMappingOverTheApi.test_rename_that_collides_with_another_column_is_an_error` | integration | — | -| `test_all_jobs_install_dev_deps` | — | — | +| `HeaderMappingOverTheApi.test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | integration | — | -| `test_g2_row_exists` | — | — | +| `test_no_mapping_returns_headers_unchanged` | — | — | -| `test_g2_is_accepted_or_deferred` | — | — | +| `test_rename_replaces_only_the_named_header` | — | — | -| `test_g2_section_exists` | — | — | +| `test_blank_rename_keeps_the_original` | — | — | -| `test_g2_is_not_marked_closed` | — | — | +| `test_unknown_original_is_rejected` | — | — | -## `tests/test_csv_loader_arbitrary_shapes.py` +| `test_original_listed_twice_is_rejected` | — | — | + +| `test_rename_colliding_with_an_untouched_column_is_rejected` | — | — | + +| `test_collision_is_judged_after_sanitising` | — | — | + +| `test_swapping_two_names_is_allowed` | — | — | + +| `test_renamed_header_becomes_the_database_column` | — | — | + +| `test_upload_without_a_mapping_keeps_csv_headers` | — | — | + +| `test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | — | — | + +| `test_rename_that_collides_with_another_column_is_an_error` | — | — | + +| `test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | — | — | + +## `tests/system_integration/test_import_counts.py` | Test | Markers | Objective | |---|---|---| -| `CsvLoaderArbitraryShapes.test_shape_tiny` | — | — | +| `ImportCountsArePersisted.test_dynamic_import_persists_all_four_counts` | integration | — | -| `CsvLoaderArbitraryShapes.test_shape_medium` | — | — | +| `ImportCountsArePersisted.test_te_import_persists_counts` | integration | — | -| `CsvLoaderArbitraryShapes.test_shape_skinny` | — | — | +| `test_dynamic_import_persists_all_four_counts` | — | — | -| `test_shape_tiny` | — | — | +| `test_te_import_persists_counts` | — | — | -| `test_shape_medium` | — | — | +## `tests/system_integration/test_migrations.py` -| `test_shape_skinny` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `MigrationsOnScratchSchema.test_upgrade_head_creates_tables_and_unique_constraints` | integration | — | -## `tests/test_csv_utilise.py` +| `MigrationsOnScratchSchema.test_control_schema_is_created_and_removed` | integration | — | + +| `MigrationsOnScratchSchema.test_downgrade_reverses_each_step` | integration | — | + +| `MigrationsOnScratchSchema.test_baseline_adopts_a_fallback_created_schema` | integration | — | + +| `MigrationsOnScratchSchema.test_existing_duplicates_stop_the_upgrade_with_remediation` | integration | — | + +| `RegistryUniqueness.test_bootstrap_brings_the_schema_to_head` | integration | — | + +| `RegistryUniqueness.test_duplicate_hash_is_rejected_by_the_database` | integration | — | + +| `RegistryUniqueness.test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | integration | — | + +| `RegistryUniqueness.test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | integration | — | + +| `test_upgrade_head_creates_tables_and_unique_constraints` | — | — | + +| `test_control_schema_is_created_and_removed` | — | — | + +| `test_downgrade_reverses_each_step` | — | — | + +| `test_baseline_adopts_a_fallback_created_schema` | — | — | + +| `test_existing_duplicates_stop_the_upgrade_with_remediation` | — | — | + +| `test_bootstrap_brings_the_schema_to_head` | — | — | + +| `test_duplicate_hash_is_rejected_by_the_database` | — | — | + +| `test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | — | — | + +| `test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | — | — | + +## `tests/system_integration/test_oidc.py` | Test | Markers | Objective | |---|---|---| -| `CsvUtiliseArgumentParsing.test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | +| `TokenVerification.test_valid_token_yields_the_username` | unit | — | -| `CsvUtiliseArgumentParsing.test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | +| `TokenVerification.test_rejected_tokens` | unit | — | -| `CsvUtiliseArgumentParsing.test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | +| `OidcSignIn.test_valid_token_for_a_member_gets_a_session` | integration | — | -| `CsvUtiliseArgumentParsing.test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | +| `OidcSignIn.test_valid_token_but_no_membership_or_account_is_401` | integration | — | -| `CsvUtiliseArgumentParsing.test_export_requires_two_positional_args` | — | `export` needs both and → exit 1 otherwise. | +| `OidcSignIn.test_invalid_token_is_401` | integration | — | -| `CsvUtiliseArgumentParsing.test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | +| `OidcSignIn.test_inactive_user_is_401` | integration | — | -| `CsvUtiliseArgumentParsing.test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | +| `OidcSignIn.test_route_is_404_when_oidc_is_not_configured` | integration | — | -| `CsvUtiliseArgumentParsing.test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | +| `ConfigFailsClosed.test_off_by_default_and_complete_config_passes` | unit | — | -| `CsvUtiliseArgumentParsing.test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | +| `ConfigFailsClosed.test_missing_audience_or_insecure_jwks_url_refuses_to_start` | unit | — | -| `test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | +| `test_valid_token_yields_the_username` | — | — | -| `test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | +| `test_rejected_tokens` | — | — | -| `test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | +| `test_valid_token_for_a_member_gets_a_session` | — | — | -| `test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | +| `test_valid_token_but_no_membership_or_account_is_401` | — | — | -| `test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | +| `test_invalid_token_is_401` | — | — | -| `test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | +| `test_inactive_user_is_401` | — | — | -| `test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | +| `test_route_is_404_when_oidc_is_not_configured` | — | — | -| `test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | +| `test_off_by_default_and_complete_config_passes` | — | — | -| `test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | +| `test_missing_audience_or_insecure_jwks_url_refuses_to_start` | — | — | -## `tests/test_csv_validator.py` +## `tests/system_integration/test_original_bytes.py` | Test | Markers | Objective | |---|---|---| -| `CsvValidatorTests.test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | +| `OriginalBytesAreKept.test_bom_prefixed_csv_round_trips_byte_identical` | integration | — | -| `CsvValidatorTests.test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | +| `OriginalBytesAreKept.test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | integration | — | -| `CsvValidatorTests.test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | +| `OriginalBytesAreKept.test_the_decoded_view_still_feeds_the_loader_and_source_rows` | integration | — | -| `CsvValidatorTests.test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | +| `OriginalBytesAreKept.test_a_failed_upload_keeps_the_raw_original_too` | integration | — | -| `test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | +| `OriginalBytesAreKept.test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | integration | — | -| `test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | +| `OriginalBytesAreKept.test_invalid_base64_is_rejected_before_anything_loads` | integration | — | -| `test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | +| `test_bom_prefixed_csv_round_trips_byte_identical` | — | — | -| `test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | +| `test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | — | — | -## `tests/test_e2e_pipeline.py` +| `test_the_decoded_view_still_feeds_the_loader_and_source_rows` | — | — | + +| `test_a_failed_upload_keeps_the_raw_original_too` | — | — | + +| `test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | — | — | + +| `test_invalid_base64_is_rejected_before_anything_loads` | — | — | + +## `tests/system_integration/test_parity.py` | Test | Markers | Objective | |---|---|---| -| `E2EPipelineValidateOnly.test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | +| `TestDevSeedCounts.test_dev_seed_row_counts` | — | — | -| `E2EPipelineValidateOnly.test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | +| `TestAllEnvironmentsHaveRequiredTables.test_dev_has_required_tables` | — | — | -| `E2EPipelineValidateOnly.test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | +| `TestAllEnvironmentsHaveRequiredTables.test_test_has_required_tables` | — | — | -| `E2EPipelineValidateOnly.test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | +| `TestAllEnvironmentsHaveRequiredTables.test_staging_has_required_tables` | — | — | -| `E2EPipelineWithDatabase.test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | +| `TestAllEnvironmentsHaveRequiredTables.test_prod_has_required_tables` | — | — | -| `E2EPipelineWithDatabase.test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | +| `TestNoSeedInStagingAndProd.test_staging_has_no_seed_data` | — | — | -| `test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | +| `TestNoSeedInStagingAndProd.test_prod_has_no_seed_data` | — | — | -| `test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | +| `TestIdempotentDeployParity.test_dev_row_counts_stable_after_second_deploy` | — | — | -| `test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | +| `test_dev_seed_row_counts` | — | — | -| `test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | +| `test_dev_has_required_tables` | — | — | -| `test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | +| `test_test_has_required_tables` | — | — | + +| `test_staging_has_required_tables` | — | — | + +| `test_prod_has_required_tables` | — | — | + +| `test_staging_has_no_seed_data` | — | — | + +| `test_prod_has_no_seed_data` | — | — | + +| `test_dev_row_counts_stable_after_second_deploy` | — | — | + +## `tests/system_integration/test_password_reset.py` + +| Test | Markers | Objective | +|---|---|---| +| `ChangeOwnPassword.test_new_password_works_old_does_not_and_other_sessions_end` | integration | — | + +| `ChangeOwnPassword.test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | integration | — | + +| `ChangeOwnPassword.test_short_new_password_is_rejected` | integration | — | + +| `ChangeOwnPassword.test_needs_a_session` | integration | — | + +| `AdminReset.test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | integration | — | + +| `AdminReset.test_reset_is_written_to_the_tenants_audit_log` | integration | — | + +| `AdminReset.test_non_admin_cannot_reset` | integration | — | + +| `AdminReset.test_admin_cannot_reset_a_user_in_another_tenant` | integration | — | + +| `AdminReset.test_user_who_is_also_in_another_tenant_is_refused` | integration | — | + +| `BreakGlassReset.test_resets_password_clears_lockout_and_ends_sessions` | integration | — | + +| `BreakGlassReset.test_unknown_user_and_short_password_fail_without_changes` | integration | — | + +| `test_new_password_works_old_does_not_and_other_sessions_end` | — | — | + +| `test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | — | — | + +| `test_short_new_password_is_rejected` | — | — | + +| `test_needs_a_session` | — | — | + +| `test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | — | — | + +| `test_reset_is_written_to_the_tenants_audit_log` | — | — | + +| `test_non_admin_cannot_reset` | — | — | + +| `test_admin_cannot_reset_a_user_in_another_tenant` | — | — | + +| `test_user_who_is_also_in_another_tenant_is_refused` | — | — | + +| `test_resets_password_clears_lockout_and_ends_sessions` | — | — | + +| `test_unknown_user_and_short_password_fail_without_changes` | — | — | + +## `tests/system_integration/test_preserved_uploads.py` + +| Test | Markers | Objective | +|---|---|---| +| `PreservedUploads.test_original_bytes_and_rows_are_kept_with_the_outcome` | integration | — | + +| `PreservedUploads.test_a_failed_upload_still_keeps_its_original` | integration | — | + +| `PreservedUploads.test_failure_reasons_and_log_are_readable_from_another_session` | integration | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | + +| `PreservedUploads.test_partly_failed_and_duplicate_outcomes` | integration | — | + +| `PreservedUploads.test_overwrite_keeps_the_replaced_original` | integration | — | + +| `PreservedUploads.test_preserved_tables_reject_update_and_delete` | integration | — | + +| `PreservedUploads.test_unknown_upload_is_404` | integration | — | + +| `test_original_bytes_and_rows_are_kept_with_the_outcome` | — | — | + +| `test_a_failed_upload_still_keeps_its_original` | — | — | + +| `test_failure_reasons_and_log_are_readable_from_another_session` | — | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | + +| `test_partly_failed_and_duplicate_outcomes` | — | — | + +| `test_overwrite_keeps_the_replaced_original` | — | — | + +| `test_preserved_tables_reject_update_and_delete` | — | — | + +| `test_unknown_upload_is_404` | — | — | + +## `tests/system_integration/test_ragged_rows.py` + +| Test | Markers | Objective | +|---|---|---| +| `ExtraCells.test_non_empty_cells_beyond_the_header_are_returned` | unit | — | + +| `ExtraCells.test_trailing_empty_cells_are_not_extra` | unit | — | + +| `ExtraCells.test_short_rows_have_no_extra_cells` | unit | — | + +| `RaggedRowsOverTheApi.test_dynamic_mode_reports_a_row_wider_than_the_header` | integration | — | + +| `RaggedRowsOverTheApi.test_dynamic_mode_accepts_a_trailing_empty_cell` | integration | — | + +| `RaggedRowsOverTheApi.test_te_mode_reports_a_row_wider_than_the_header` | integration | — | + +| `RaggedRowsOverTheApi.test_multiple_ragged_rows_each_get_a_reason` | integration | Every ragged row gets its own entry in rowErrors with a reason. | + +| `RaggedRowsOverTheApi.test_dynamic_mode_accounts_for_every_row_with_a_stored_reason` | integration | — | + +| `RaggedRowsOverTheApi.test_te_mode_accounts_for_every_row_with_a_stored_reason` | integration | — | + +| `test_non_empty_cells_beyond_the_header_are_returned` | — | — | + +| `test_trailing_empty_cells_are_not_extra` | — | — | + +| `test_short_rows_have_no_extra_cells` | — | — | + +| `test_dynamic_mode_reports_a_row_wider_than_the_header` | — | — | + +| `test_dynamic_mode_accepts_a_trailing_empty_cell` | — | — | + +| `test_te_mode_reports_a_row_wider_than_the_header` | — | — | + +| `test_multiple_ragged_rows_each_get_a_reason` | — | Every ragged row gets its own entry in rowErrors with a reason. | + +| `test_dynamic_mode_accounts_for_every_row_with_a_stored_reason` | — | — | + +| `test_te_mode_accounts_for_every_row_with_a_stored_reason` | — | — | + +## `tests/system_integration/test_registry_race.py` + +| Test | Markers | Objective | +|---|---|---| +| `test_identical_file_uploaded_concurrently_registers_once` | — | — | + +| `test_identical_content_under_different_names_registers_once` | — | The case where file_hash, not file_name, is the guard: nothing about the | + +| `test_same_name_different_content_uploaded_concurrently_registers_once` | — | — | + +## `tests/system_integration/test_review.py` + +| Test | Markers | Objective | +|---|---|---| +| `Review.test_pages_through_every_row_with_validation` | integration | — | + +| `Review.test_validates_rows_beyond_the_reported_error_cap` | integration | — | + +| `Review.test_row_decisions_record_actor_and_reason_and_the_latest_wins` | integration | — | + +| `Review.test_cell_edits_are_versions_over_the_unchanged_original` | integration | — | + +| `Review.test_refusals` | integration | — | + +| `Review.test_te_review_shows_the_database_reason_a_row_was_rejected` | integration | — | + +| `Review.test_changes_are_audited_and_insert_only` | integration | — | + +| `test_pages_through_every_row_with_validation` | — | — | + +| `test_validates_rows_beyond_the_reported_error_cap` | — | — | + +| `test_row_decisions_record_actor_and_reason_and_the_latest_wins` | — | — | + +| `test_cell_edits_are_versions_over_the_unchanged_original` | — | — | + +| `test_refusals` | — | — | + +| `test_te_review_shows_the_database_reason_a_row_was_rejected` | — | — | + +| `test_changes_are_audited_and_insert_only` | — | — | + +## `tests/system_integration/test_review_cases.py` + +| Test | Markers | Objective | +|---|---|---| +| `ReviewCases.test_P02_paging_bounds` | integration | — | + +| `ReviewCases.test_P04_ragged_row_is_flagged` | integration | — | + +| `ReviewCases.test_P05_a_failed_upload_can_be_reviewed` | integration | — | + +| `ReviewCases.test_P08_an_edit_to_a_bad_value_is_flagged` | integration | — | + +| `ReviewCases.test_P09_edit_by_column_index_and_to_an_empty_value` | integration | — | + +| `ReviewCases.test_P10_reason_is_stored_trimmed` | integration | — | + +| `ReviewCases.test_P11_actor_is_the_signed_in_user` | integration | — | + +| `ReviewCases.test_P12_every_role_can_view_rows_and_history` | integration | — | + +| `ReviewCases.test_N05_row_numbers_outside_the_upload` | integration | — | -| `test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | +| `ReviewCases.test_N06_column_index_out_of_range` | integration | — | -## `tests/test_evals_runner.py` +| `ReviewCases.test_N07_unknown_upload_on_every_route` | integration | — | -| Test | Markers | Objective | -|---|---|---| -| `EvalsRunnerTests.test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | +| `ReviewCases.test_N08_N09_uploader_and_auditor_cannot_change` | integration | — | -| `EvalsRunnerTests.test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | +| `ReviewCases.test_N10_an_invalid_session_is_refused` | integration | — | -| `EvalsRunnerTests.test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | +| `ReviewCases.test_N11_another_tenant_cannot_see_or_change_the_upload` | integration | — | -| `EvalsRunnerTests.test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | +| `ReviewCases.test_N12_missing_or_wrong_body_fields` | integration | — | -| `EvalsRunnerTests.test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | +| `ReviewCases.test_N13_review_tables_reject_update_delete_and_truncate` | integration | — | -| `EvalsRunnerTests.test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | +| `ReviewCases.test_N14_database_rejects_a_blank_reason` | integration | — | -| `EvalsRunnerTests.test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | +| `ReviewCases.test_N16_a_non_numeric_upload_id` | integration | — | -| `test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | +| `test_P02_paging_bounds` | — | — | -| `test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | +| `test_P04_ragged_row_is_flagged` | — | — | -| `test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | +| `test_P05_a_failed_upload_can_be_reviewed` | — | — | -| `test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | +| `test_P08_an_edit_to_a_bad_value_is_flagged` | — | — | -| `test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | +| `test_P09_edit_by_column_index_and_to_an_empty_value` | — | — | -| `test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | +| `test_P10_reason_is_stored_trimmed` | — | — | -| `test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | +| `test_P11_actor_is_the_signed_in_user` | — | — | -## `tests/test_evals_tier_e_x.py` +| `test_P12_every_role_can_view_rows_and_history` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `TierEDiffSignatures.test_identical_schemas_report_no_differences` | unit | — | +| `test_N05_row_numbers_outside_the_upload` | — | — | -| `TierEDiffSignatures.test_namespace_is_not_part_of_the_comparison` | unit | — | +| `test_N06_column_index_out_of_range` | — | — | -| `TierEDiffSignatures.test_missing_table_is_caught` | unit | — | +| `test_N07_unknown_upload_on_every_route` | — | — | -| `TierEDiffSignatures.test_extra_table_is_caught` | unit | — | +| `test_N08_N09_uploader_and_auditor_cannot_change` | — | — | -| `TierEDiffSignatures.test_added_column_is_caught` | unit | — | +| `test_N10_an_invalid_session_is_refused` | — | — | -| `TierEDiffSignatures.test_changed_type_is_caught` | unit | — | +| `test_N11_another_tenant_cannot_see_or_change_the_upload` | — | — | -| `TierEDiffSignatures.test_changed_nullability_is_caught` | unit | — | +| `test_N12_missing_or_wrong_body_fields` | — | — | -| `TierEDiffSignatures.test_reordered_columns_are_caught_as_drift` | unit | — | +| `test_N13_review_tables_reject_update_delete_and_truncate` | — | — | -| `TierEDiffSignatures.test_env_targets_cover_all_four_environments` | unit | — | +| `test_N14_database_rejects_a_blank_reason` | — | — | -| `TierEColumnSignatureQuery.test_query_is_sent_on_stdin_not_via_dash_c` | unit | — | +| `test_N16_a_non_numeric_upload_id` | — | — | -| `TierEColumnSignatureQuery.test_query_failure_is_reported_as_a_query_error` | unit | — | +## `tests/system_integration/test_roles.py` -| `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | +| Test | Markers | Objective | +|---|---|---| +| `MatrixIsDocumented.test_docs_table_matches_the_enforced_permissions` | unit | — | -| `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | +| `MatrixIsDocumented.test_every_action_has_a_route_and_only_known_roles` | unit | — | -| `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | +| `RoleMatrixOverTheApi.test_every_role_against_every_route` | integration | — | -| `TierXSqliteStructure.test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | unit | — | +| `RoleMatrixOverTheApi.test_no_session_requests_are_not_role_checked` | integration | — | -| `TierXSqliteStructure.test_structure_is_read_back_from_executed_ddl` | unit | — | +| `test_docs_table_matches_the_enforced_permissions` | — | — | -| `TierXSqliteStructure.test_invalid_ddl_is_reported_rather_than_raised` | unit | — | +| `test_every_action_has_a_route_and_only_known_roles` | — | — | -| `TierXSqliteStructure.test_sqlite_internal_tables_are_excluded` | unit | — | +| `test_every_role_against_every_route` | — | — | -| `TierXSqliteStructure.test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | unit | — | +| `test_no_session_requests_are_not_role_checked` | — | — | -| `TierXSqliteStructure.test_known_missing_tables_are_genuinely_absent` | unit | — | +## `tests/system_integration/test_runtime_role.py` -| `TierXSqliteStructure.test_missing_plus_present_accounts_for_all_twelve_tables` | unit | — | +| Test | Markers | Objective | +|---|---|---| +| `RuntimeRole.test_script_keeps_the_password_out_of_its_output` | integration | — | -| `TierRegistration.test_all_five_tiers_are_registered` | unit | — | +| `RuntimeRole.test_the_api_works_as_the_runtime_role` | integration | — | -| `test_identical_schemas_report_no_differences` | — | — | +| `RuntimeRole.test_runtime_role_cannot_change_or_unprotect_append_only_tables` | integration | — | -| `test_namespace_is_not_part_of_the_comparison` | — | — | +| `test_script_keeps_the_password_out_of_its_output` | — | — | -| `test_missing_table_is_caught` | — | — | +| `test_the_api_works_as_the_runtime_role` | — | — | -| `test_extra_table_is_caught` | — | — | +| `test_runtime_role_cannot_change_or_unprotect_append_only_tables` | — | — | -| `test_added_column_is_caught` | — | — | +## `tests/system_integration/test_te_loader_integration.py` -| `test_changed_type_is_caught` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `TeUploadOrganisations.test_valid_te_upload_returns_ok` | integration | A well-formed T&E CSV must return status=ok. | -| `test_changed_nullability_is_caught` | — | — | +| `TeUploadOrganisations.test_inserted_rows_count_matches_data_rows` | integration | insertedRows must equal the number of data rows supplied. | -| `test_reordered_columns_are_caught_as_drift` | — | — | +| `TeUploadOrganisations.test_table_name_in_response_is_schema_qualified` | integration | tableName must be schema-qualified (e.g. te_dev.organisations). | -| `test_env_targets_cover_all_four_environments` | — | — | +| `TeUploadOrganisations.test_rows_are_queryable_after_upload` | integration | Inserted rows must appear in the T&E table row count. | -| `test_query_is_sent_on_stdin_not_via_dash_c` | — | — | +| `TeUploadOrganisations.test_core_te_tables_undisturbed_after_upload` | integration | A T&E upload must not drop or alter the other 11 core tables. | -| `test_query_failure_is_reported_as_a_query_error` | — | — | +| `TeUploadOrganisations.test_reupload_cleans_prior_registry_entry` | integration | Re-uploading the same file name must not duplicate registry entries. | -| `test_genuinely_empty_schema_still_fails` | — | — | +| `TeUploadOrganisations.test_invalid_target_table_is_rejected` | integration | A targetTable not in TE_TABLES must return status=error. | -| `test_placeholders_are_substituted` | — | — | +| `TeUploadOrganisations.test_empty_file_is_rejected` | integration | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | -| `test_every_known_table_has_a_substitution` | — | — | +| `TeUploadOrganisations.test_header_only_file_is_rejected` | integration | A header-only CSV (no data rows) must return status=invalid_structure. | -| `test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | — | — | +| `TeUploadOrganisations.test_unknown_columns_are_rejected` | integration | Columns not in the target table must return status=error. | -| `test_structure_is_read_back_from_executed_ddl` | — | — | +| `TeUploadOrganisations.test_missing_target_table_param_is_rejected` | integration | mode=te with no targetTable must return 422 from the API layer. | -| `test_invalid_ddl_is_reported_rather_than_raised` | — | — | +| `DynamicOverwriteOfTeOriginFile.test_dynamic_overwrite_of_te_origin_entry_succeeds` | integration | A dynamic-mode overwrite of a filename previously registered in | -| `test_sqlite_internal_tables_are_excluded` | — | — | +| `test_valid_te_upload_returns_ok` | — | A well-formed T&E CSV must return status=ok. | -| `test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | — | — | +| `test_inserted_rows_count_matches_data_rows` | — | insertedRows must equal the number of data rows supplied. | -| `test_known_missing_tables_are_genuinely_absent` | — | — | +| `test_table_name_in_response_is_schema_qualified` | — | tableName must be schema-qualified (e.g. te_dev.organisations). | -| `test_missing_plus_present_accounts_for_all_twelve_tables` | — | — | +| `test_rows_are_queryable_after_upload` | — | Inserted rows must appear in the T&E table row count. | -| `test_all_five_tiers_are_registered` | — | — | +| `test_core_te_tables_undisturbed_after_upload` | — | A T&E upload must not drop or alter the other 11 core tables. | -## `tests/test_failed_uploads.py` +| `test_reupload_cleans_prior_registry_entry` | — | Re-uploading the same file name must not duplicate registry entries. | -| Test | Markers | Objective | -|---|---|---| -| `EveryRowFailed.test_every_row_failing_is_an_error_and_saves_nothing` | integration | — | +| `test_invalid_target_table_is_rejected` | — | A targetTable not in TE_TABLES must return status=error. | -| `EveryRowFailed.test_failed_overwrite_keeps_the_previous_upload` | integration | — | +| `test_empty_file_is_rejected` | — | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | -| `EveryRowFailed.test_partly_failed_upload_still_imports_and_reports_the_failures` | integration | — | +| `test_header_only_file_is_rejected` | — | A header-only CSV (no data rows) must return status=invalid_structure. | -| `test_every_row_failing_is_an_error_and_saves_nothing` | — | — | +| `test_unknown_columns_are_rejected` | — | Columns not in the target table must return status=error. | -| `test_failed_overwrite_keeps_the_previous_upload` | — | — | +| `test_missing_target_table_param_is_rejected` | — | mode=te with no targetTable must return 422 from the API layer. | -| `test_partly_failed_upload_still_imports_and_reports_the_failures` | — | — | +| `test_dynamic_overwrite_of_te_origin_entry_succeeds` | — | A dynamic-mode overwrite of a filename previously registered in | -## `tests/test_header_mapping.py` +## `tests/system_integration/test_te_reupload_registry.py` | Test | Markers | Objective | |---|---|---| -| `ApplyHeaderMapping.test_no_mapping_returns_headers_unchanged` | unit | — | +| `TeReuploadRegistry.test_all_failed_reupload_keeps_the_earlier_registry_record` | integration | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | -| `ApplyHeaderMapping.test_rename_replaces_only_the_named_header` | unit | — | +| `TeReuploadRegistry.test_reupload_that_inserts_rows_still_replaces_the_registry_record` | integration | A T&E re-upload that loads at least one row still replaces the registry record. | -| `ApplyHeaderMapping.test_blank_rename_keeps_the_original` | unit | — | +| `TeReuploadRegistry.test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | integration | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | -| `ApplyHeaderMapping.test_unknown_original_is_rejected` | unit | — | +| `test_all_failed_reupload_keeps_the_earlier_registry_record` | — | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | -| `ApplyHeaderMapping.test_original_listed_twice_is_rejected` | unit | — | +| `test_reupload_that_inserts_rows_still_replaces_the_registry_record` | — | A T&E re-upload that loads at least one row still replaces the registry record. | -| `ApplyHeaderMapping.test_rename_colliding_with_an_untouched_column_is_rejected` | unit | — | +| `test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | — | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | -| `ApplyHeaderMapping.test_collision_is_judged_after_sanitising` | unit | — | +## `tests/system_integration/test_tenant_isolation.py` -| `ApplyHeaderMapping.test_swapping_two_names_is_allowed` | unit | — | +| Test | Markers | Objective | +|---|---|---| +| `CrossTenantIsolation.test_upload_lands_in_the_tenants_own_schema` | integration | — | -| `HeaderMappingOverTheApi.test_renamed_header_becomes_the_database_column` | integration | — | +| `CrossTenantIsolation.test_other_tenant_cannot_list_read_or_delete` | integration | — | -| `HeaderMappingOverTheApi.test_upload_without_a_mapping_keeps_csv_headers` | integration | — | +| `CrossTenantIsolation.test_requests_without_a_session_see_no_tenant_data` | integration | — | -| `HeaderMappingOverTheApi.test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | integration | — | +| `CrossTenantIsolation.test_duplicate_detection_is_per_tenant` | integration | — | -| `HeaderMappingOverTheApi.test_rename_that_collides_with_another_column_is_an_error` | integration | — | +| `CrossTenantIsolation.test_audit_log_is_per_tenant` | integration | — | -| `HeaderMappingOverTheApi.test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | integration | — | +| `CrossTenantIsolation.test_te_mode_loads_into_the_tenants_own_te_tables` | integration | — | -| `test_no_mapping_returns_headers_unchanged` | — | — | +| `CrossTenantIsolation.test_te_table_counts_are_per_tenant` | integration | — | -| `test_rename_replaces_only_the_named_header` | — | — | +| `CrossTenantIsolation.test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | integration | — | -| `test_blank_rename_keeps_the_original` | — | — | +| `CrossTenantIsolation.test_invalid_session_is_401_not_a_fallback_to_default` | integration | — | -| `test_unknown_original_is_rejected` | — | — | +| `CrossTenantIsolation.test_tenant_schema_is_at_head_with_its_own_registry` | integration | — | -| `test_original_listed_twice_is_rejected` | — | — | +| `CrossTenantIsolation.test_concurrent_requests_never_see_another_tenants_schema` | integration | — | -| `test_rename_colliding_with_an_untouched_column_is_rejected` | — | — | +| `CrossTenantIsolation.test_provisioning_is_idempotent` | integration | — | -| `test_collision_is_judged_after_sanitising` | — | — | +| `test_upload_lands_in_the_tenants_own_schema` | — | — | -| `test_swapping_two_names_is_allowed` | — | — | +| `test_other_tenant_cannot_list_read_or_delete` | — | — | -| `test_renamed_header_becomes_the_database_column` | — | — | +| `test_requests_without_a_session_see_no_tenant_data` | — | — | -| `test_upload_without_a_mapping_keeps_csv_headers` | — | — | +| `test_duplicate_detection_is_per_tenant` | — | — | -| `test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | — | — | +| `test_audit_log_is_per_tenant` | — | — | -| `test_rename_that_collides_with_another_column_is_an_error` | — | — | +| `test_te_mode_loads_into_the_tenants_own_te_tables` | — | — | -| `test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | — | — | +| `test_te_table_counts_are_per_tenant` | — | — | -## `tests/test_import_counts.py` +| `test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `ImportCountsArePersisted.test_dynamic_import_persists_all_four_counts` | integration | — | +| `test_invalid_session_is_401_not_a_fallback_to_default` | — | — | -| `ImportCountsArePersisted.test_te_import_persists_counts` | integration | — | +| `test_tenant_schema_is_at_head_with_its_own_registry` | — | — | -| `test_dynamic_import_persists_all_four_counts` | — | — | +| `test_concurrent_requests_never_see_another_tenants_schema` | — | — | -| `test_te_import_persists_counts` | — | — | +| `test_provisioning_is_idempotent` | — | — | -## `tests/test_issue_04_multi_file_upload.py` +## `tests/uat/test_issue_04_multi_file_upload.py` | Test | Markers | Objective | |---|---|---| @@ -921,7 +1081,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_a_larger_file_loads_completely` | — | Row count is not silently truncated. | -## `tests/test_issue_05_import_summary.py` +## `tests/uat/test_issue_05_import_summary.py` | Test | Markers | Objective | |---|---|---| @@ -961,479 +1121,455 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_counts_are_per_batch_not_cumulative` | — | A second, larger upload must report its own totals, not a running sum. | -## `tests/test_migrations.py` +## `tests/unit/test_api_hardening.py` | Test | Markers | Objective | |---|---|---| -| `MigrationsOnScratchSchema.test_upgrade_head_creates_tables_and_unique_constraints` | integration | — | - -| `MigrationsOnScratchSchema.test_control_schema_is_created_and_removed` | integration | — | - -| `MigrationsOnScratchSchema.test_downgrade_reverses_each_step` | integration | — | +| `AuthFailsClosed.test_dev_may_run_without_a_key` | unit | — | -| `MigrationsOnScratchSchema.test_baseline_adopts_a_fallback_created_schema` | integration | — | +| `AuthFailsClosed.test_any_key_is_accepted_in_dev` | unit | — | -| `MigrationsOnScratchSchema.test_existing_duplicates_stop_the_upgrade_with_remediation` | integration | — | +| `AuthFailsClosed.test_prod_without_a_key_refuses_to_start` | unit | — | -| `RegistryUniqueness.test_bootstrap_brings_the_schema_to_head` | integration | — | +| `AuthFailsClosed.test_staging_and_test_also_refuse` | unit | — | -| `RegistryUniqueness.test_duplicate_hash_is_rejected_by_the_database` | integration | — | +| `AuthFailsClosed.test_deployed_env_with_a_key_starts` | unit | — | -| `RegistryUniqueness.test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | integration | — | +| `RateLimiterAlgorithm.test_requests_under_budget_are_allowed` | unit | — | -| `RegistryUniqueness.test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | integration | — | +| `RateLimiterAlgorithm.test_request_over_budget_is_refused` | unit | — | -| `test_upgrade_head_creates_tables_and_unique_constraints` | — | — | +| `RateLimiterAlgorithm.test_retry_after_points_at_the_oldest_hit_leaving_the_window` | unit | — | -| `test_control_schema_is_created_and_removed` | — | — | +| `RateLimiterAlgorithm.test_window_expiry_frees_the_budget` | unit | — | -| `test_downgrade_reverses_each_step` | — | — | +| `RateLimiterAlgorithm.test_zero_budget_disables_the_limit` | unit | — | -| `test_baseline_adopts_a_fallback_created_schema` | — | — | +| `RateLimiterAlgorithm.test_read_and_write_scopes_are_independent` | unit | — | -| `test_existing_duplicates_stop_the_upgrade_with_remediation` | — | — | +| `RateLimiterAlgorithm.test_clients_are_independent` | unit | — | -| `test_bootstrap_brings_the_schema_to_head` | — | — | +| `RateLimiterAlgorithm.test_bucket_table_is_bounded` | unit | — | -| `test_duplicate_hash_is_rejected_by_the_database` | — | — | +| `RateLimiterAlgorithm.test_reset_clears_state` | unit | — | -| `test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | — | — | +| `RateLimitMiddleware.test_exceeding_the_write_budget_returns_429_with_retry_after` | unit | — | -| `test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | — | — | +| `RateLimitMiddleware.test_health_is_never_rate_limited` | unit | — | -## `tests/test_oidc.py` +| `RateLimitMiddleware.test_openapi_is_never_rate_limited` | unit | — | -| Test | Markers | Objective | -|---|---|---| -| `TokenVerification.test_valid_token_yields_the_username` | unit | — | +| `RateLimitMiddleware.test_disabled_budget_allows_everything` | unit | — | -| `TokenVerification.test_rejected_tokens` | unit | — | +| `RateLimitMiddleware.test_forwarded_for_cannot_be_used_to_evade_the_limit` | unit | — | -| `OidcSignIn.test_valid_token_for_a_member_gets_a_session` | integration | — | +| `AuditWriterContract.test_only_known_actions_are_accepted` | unit | — | -| `OidcSignIn.test_valid_token_but_no_membership_or_account_is_401` | integration | — | +| `AuditWriterContract.test_unknown_action_raises_before_any_sql_runs` | unit | — | -| `OidcSignIn.test_invalid_token_is_401` | integration | — | +| `AuditWriterContract.test_valid_action_reaches_the_cursor_with_bound_parameters` | unit | — | -| `OidcSignIn.test_inactive_user_is_401` | integration | — | +| `AuditWriterContract.test_file_id_is_coerced_from_the_loaders_string_form` | unit | — | -| `OidcSignIn.test_route_is_404_when_oidc_is_not_configured` | integration | — | +| `AuditWriterContract.test_unparseable_file_id_degrades_to_null_rather_than_raising` | unit | — | -| `ConfigFailsClosed.test_off_by_default_and_complete_config_passes` | unit | — | +| `test_dev_may_run_without_a_key` | — | — | -| `ConfigFailsClosed.test_missing_audience_or_insecure_jwks_url_refuses_to_start` | unit | — | +| `test_any_key_is_accepted_in_dev` | — | — | -| `test_valid_token_yields_the_username` | — | — | +| `test_prod_without_a_key_refuses_to_start` | — | — | -| `test_rejected_tokens` | — | — | +| `test_staging_and_test_also_refuse` | — | — | -| `test_valid_token_for_a_member_gets_a_session` | — | — | +| `test_deployed_env_with_a_key_starts` | — | — | -| `test_valid_token_but_no_membership_or_account_is_401` | — | — | +| `test_requests_under_budget_are_allowed` | — | — | -| `test_invalid_token_is_401` | — | — | +| `test_request_over_budget_is_refused` | — | — | -| `test_inactive_user_is_401` | — | — | +| `test_retry_after_points_at_the_oldest_hit_leaving_the_window` | — | — | -| `test_route_is_404_when_oidc_is_not_configured` | — | — | +| `test_window_expiry_frees_the_budget` | — | — | -| `test_off_by_default_and_complete_config_passes` | — | — | +| `test_zero_budget_disables_the_limit` | — | — | -| `test_missing_audience_or_insecure_jwks_url_refuses_to_start` | — | — | +| `test_read_and_write_scopes_are_independent` | — | — | -## `tests/test_original_bytes.py` +| `test_clients_are_independent` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `OriginalBytesAreKept.test_bom_prefixed_csv_round_trips_byte_identical` | integration | — | +| `test_bucket_table_is_bounded` | — | — | -| `OriginalBytesAreKept.test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | integration | — | +| `test_reset_clears_state` | — | — | -| `OriginalBytesAreKept.test_the_decoded_view_still_feeds_the_loader_and_source_rows` | integration | — | +| `test_exceeding_the_write_budget_returns_429_with_retry_after` | — | — | -| `OriginalBytesAreKept.test_a_failed_upload_keeps_the_raw_original_too` | integration | — | +| `test_health_is_never_rate_limited` | — | — | -| `OriginalBytesAreKept.test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | integration | — | +| `test_openapi_is_never_rate_limited` | — | — | -| `OriginalBytesAreKept.test_invalid_base64_is_rejected_before_anything_loads` | integration | — | +| `test_disabled_budget_allows_everything` | — | — | -| `test_bom_prefixed_csv_round_trips_byte_identical` | — | — | +| `test_forwarded_for_cannot_be_used_to_evade_the_limit` | — | — | -| `test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | — | — | +| `test_only_known_actions_are_accepted` | — | — | -| `test_the_decoded_view_still_feeds_the_loader_and_source_rows` | — | — | +| `test_unknown_action_raises_before_any_sql_runs` | — | — | -| `test_a_failed_upload_keeps_the_raw_original_too` | — | — | +| `test_valid_action_reaches_the_cursor_with_bound_parameters` | — | — | -| `test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | — | — | +| `test_file_id_is_coerced_from_the_loaders_string_form` | — | — | -| `test_invalid_base64_is_rejected_before_anything_loads` | — | — | +| `test_unparseable_file_id_degrades_to_null_rather_than_raising` | — | — | -## `tests/test_parity.py` +## `tests/unit/test_api_unit.py` | Test | Markers | Objective | |---|---|---| -| `TestDevSeedCounts.test_dev_seed_row_counts` | — | — | - -| `TestAllEnvironmentsHaveRequiredTables.test_dev_has_required_tables` | — | — | - -| `TestAllEnvironmentsHaveRequiredTables.test_test_has_required_tables` | — | — | - -| `TestAllEnvironmentsHaveRequiredTables.test_staging_has_required_tables` | — | — | +| `UnitCsvParse.test_sanitize_columns_no_collision_on_stem_conflict` | unit | BUG-036 regression: ["foo","foo","foo_2"] must produce three | -| `TestAllEnvironmentsHaveRequiredTables.test_prod_has_required_tables` | — | — | +| `UnitCsvParse.test_sanitize_columns_dedupes_true_duplicates` | unit | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | -| `TestNoSeedInStagingAndProd.test_staging_has_no_seed_data` | — | — | +| `UnitCsvParse.test_sanitize_columns_reserved_stems_get_suffix` | unit | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | -| `TestNoSeedInStagingAndProd.test_prod_has_no_seed_data` | — | — | +| `RequestValidation.test_preview_rejects_empty_filename` | unit | — | -| `TestIdempotentDeployParity.test_dev_row_counts_stable_after_second_deploy` | — | — | +| `RequestValidation.test_preview_rejects_empty_content` | unit | — | -| `test_dev_seed_row_counts` | — | — | +| `RequestValidation.test_preview_rejects_overlong_filename` | unit | — | -| `test_dev_has_required_tables` | — | — | +| `RequestValidation.test_upload_rejects_unknown_mode` | unit | — | -| `test_test_has_required_tables` | — | — | +| `RequestValidation.test_upload_te_mode_requires_target_table` | unit | — | -| `test_staging_has_required_tables` | — | — | +| `RequestValidation.test_oversized_upload_is_rejected` | unit | — | -| `test_prod_has_required_tables` | — | — | +| `TableNameGuards.test_non_upload_tables_are_rejected` | security, unit | — | -| `test_staging_has_no_seed_data` | — | — | +| `TableNameGuards.test_overlong_names_are_rejected` | security, unit | — | -| `test_prod_has_no_seed_data` | — | — | +| `TableNameGuards.test_identifiers_are_never_string_formatted` | security, unit | Every SQL identifier must go through psycopg2.sql.Identifier. | -| `test_dev_row_counts_stable_after_second_deploy` | — | — | +| `test_sanitize_columns_no_collision_on_stem_conflict` | — | BUG-036 regression: ["foo","foo","foo_2"] must produce three | -## `tests/test_password_reset.py` +| `test_sanitize_columns_dedupes_true_duplicates` | — | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | -| Test | Markers | Objective | -|---|---|---| -| `ChangeOwnPassword.test_new_password_works_old_does_not_and_other_sessions_end` | integration | — | +| `test_sanitize_columns_reserved_stems_get_suffix` | — | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | -| `ChangeOwnPassword.test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | integration | — | +| `test_preview_rejects_empty_filename` | — | — | -| `ChangeOwnPassword.test_short_new_password_is_rejected` | integration | — | +| `test_preview_rejects_empty_content` | — | — | -| `ChangeOwnPassword.test_needs_a_session` | integration | — | +| `test_preview_rejects_overlong_filename` | — | — | -| `AdminReset.test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | integration | — | +| `test_upload_rejects_unknown_mode` | — | — | -| `AdminReset.test_reset_is_written_to_the_tenants_audit_log` | integration | — | +| `test_upload_te_mode_requires_target_table` | — | — | -| `AdminReset.test_non_admin_cannot_reset` | integration | — | +| `test_oversized_upload_is_rejected` | — | — | -| `AdminReset.test_admin_cannot_reset_a_user_in_another_tenant` | integration | — | +| `test_non_upload_tables_are_rejected` | — | — | -| `AdminReset.test_user_who_is_also_in_another_tenant_is_refused` | integration | — | +| `test_overlong_names_are_rejected` | — | — | -| `BreakGlassReset.test_resets_password_clears_lockout_and_ends_sessions` | integration | — | +| `test_identifiers_are_never_string_formatted` | — | Every SQL identifier must go through psycopg2.sql.Identifier. | -| `BreakGlassReset.test_unknown_user_and_short_password_fail_without_changes` | integration | — | +## `tests/unit/test_ci_quality_gate.py` -| `test_new_password_works_old_does_not_and_other_sessions_end` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `TestQualityGateStructure.test_has_free_tier_job` | — | — | -| `test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | — | — | +| `TestQualityGateStructure.test_has_integration_postgres_job` | — | — | -| `test_short_new_password_is_rejected` | — | — | +| `TestQualityGateStructure.test_triggers_on_pull_request` | — | — | -| `test_needs_a_session` | — | — | +| `TestQualityGateStructure.test_triggers_on_push_to_main` | — | — | -| `test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_checkout` | — | — | -| `test_reset_is_written_to_the_tenants_audit_log` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_python_setup` | — | — | -| `test_non_admin_cannot_reset` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_artifact_upload` | — | — | -| `test_admin_cannot_reset_a_user_in_another_tenant` | — | — | +| `TestAllJobsHaveConsistentStructure.test_all_jobs_install_dev_deps` | — | — | -| `test_user_who_is_also_in_another_tenant_is_refused` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_row_exists` | — | — | -| `test_resets_password_clears_lockout_and_ends_sessions` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_is_accepted_or_deferred` | — | — | -| `test_unknown_user_and_short_password_fail_without_changes` | — | — | +| `TestGapAnalysisG2Accepted.test_g2_section_exists` | — | — | -## `tests/test_preflight_ps1.py` +| `TestGapAnalysisG2Accepted.test_g2_is_not_marked_closed` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `test_no_null_coalescing_operator` | regression, unit | `??` / `??=` is PowerShell 7+ only and fails to parse on 5.1. | +| `test_has_free_tier_job` | — | — | -| `test_no_other_powershell7_only_syntax` | regression, unit | — | +| `test_has_integration_postgres_job` | — | — | -| `test_no_utf8_bom` | regression, unit | A BOM caused a noisy line-1 diff during the fix; keep the file BOM-free. | +| `test_triggers_on_pull_request` | — | — | -| `test_port_label_is_not_hardcoded` | regression, unit | The check label must follow the configured port (was fixed at 'port 5432'). | +| `test_triggers_on_push_to_main` | — | — | -| `test_connection_defaults_resolved_once_and_reused` | regression, unit | Defaults live in variables; connection commands must use them. | +| `test_all_jobs_have_checkout` | — | — | -| `test_default_port_and_host_unchanged` | regression, unit | — | +| `test_all_jobs_have_python_setup` | — | — | -| `test_exit_code_contract_documented_and_implemented` | regression, unit | — | +| `test_all_jobs_have_artifact_upload` | — | — | -| `test_parses_under_windows_powershell` | regression | Parse with the real PowerShell parser (5.1 on stock Windows). | +| `test_all_jobs_install_dev_deps` | — | — | -| `test_runs_with_skippg_and_reports_summary` | regression | — | +| `test_g2_row_exists` | — | — | -| `test_default_port_label_when_pgport_unset` | regression | — | +| `test_g2_is_accepted_or_deferred` | — | — | -| `test_port_label_follows_pgport` | regression | — | +| `test_g2_section_exists` | — | — | -| `test_pg_checks_are_optional_never_fail` | regression | — | +| `test_g2_is_not_marked_closed` | — | — | -## `tests/test_preserved_uploads.py` +## `tests/unit/test_csv_utilise.py` | Test | Markers | Objective | |---|---|---| -| `PreservedUploads.test_original_bytes_and_rows_are_kept_with_the_outcome` | integration | — | - -| `PreservedUploads.test_a_failed_upload_still_keeps_its_original` | integration | — | - -| `PreservedUploads.test_failure_reasons_and_log_are_readable_from_another_session` | integration | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | +| `CsvUtiliseArgumentParsing.test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | -| `PreservedUploads.test_partly_failed_and_duplicate_outcomes` | integration | — | +| `CsvUtiliseArgumentParsing.test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | -| `PreservedUploads.test_overwrite_keeps_the_replaced_original` | integration | — | +| `CsvUtiliseArgumentParsing.test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | -| `PreservedUploads.test_preserved_tables_reject_update_and_delete` | integration | — | +| `CsvUtiliseArgumentParsing.test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | -| `PreservedUploads.test_unknown_upload_is_404` | integration | — | +| `CsvUtiliseArgumentParsing.test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | -| `test_original_bytes_and_rows_are_kept_with_the_outcome` | — | — | +| `CsvUtiliseArgumentParsing.test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | -| `test_a_failed_upload_still_keeps_its_original` | — | — | +| `CsvUtiliseArgumentParsing.test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | -| `test_failure_reasons_and_log_are_readable_from_another_session` | — | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | +| `CsvUtiliseArgumentParsing.test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | -| `test_partly_failed_and_duplicate_outcomes` | — | — | +| `CsvUtiliseArgumentParsing.test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | -| `test_overwrite_keeps_the_replaced_original` | — | — | +| `test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | -| `test_preserved_tables_reject_update_and_delete` | — | — | +| `test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | -| `test_unknown_upload_is_404` | — | — | +| `test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | -## `tests/test_ragged_rows.py` +| `test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | -| Test | Markers | Objective | -|---|---|---| -| `ExtraCells.test_non_empty_cells_beyond_the_header_are_returned` | unit | — | +| `test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | -| `ExtraCells.test_trailing_empty_cells_are_not_extra` | unit | — | +| `test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | -| `ExtraCells.test_short_rows_have_no_extra_cells` | unit | — | +| `test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | -| `RaggedRowsOverTheApi.test_dynamic_mode_reports_a_row_wider_than_the_header` | integration | — | +| `test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | -| `RaggedRowsOverTheApi.test_dynamic_mode_accepts_a_trailing_empty_cell` | integration | — | +| `test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | -| `RaggedRowsOverTheApi.test_te_mode_reports_a_row_wider_than_the_header` | integration | — | +## `tests/unit/test_csv_validator.py` -| `RaggedRowsOverTheApi.test_multiple_ragged_rows_each_get_a_reason` | integration | Every ragged row gets its own entry in rowErrors with a reason. | +| Test | Markers | Objective | +|---|---|---| +| `CsvValidatorTests.test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | -| `test_non_empty_cells_beyond_the_header_are_returned` | — | — | +| `CsvValidatorTests.test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | -| `test_trailing_empty_cells_are_not_extra` | — | — | +| `CsvValidatorTests.test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | -| `test_short_rows_have_no_extra_cells` | — | — | +| `CsvValidatorTests.test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | -| `test_dynamic_mode_reports_a_row_wider_than_the_header` | — | — | +| `test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | -| `test_dynamic_mode_accepts_a_trailing_empty_cell` | — | — | +| `test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | -| `test_te_mode_reports_a_row_wider_than_the_header` | — | — | +| `test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | -| `test_multiple_ragged_rows_each_get_a_reason` | — | Every ragged row gets its own entry in rowErrors with a reason. | +| `test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | -## `tests/test_registry_race.py` +## `tests/unit/test_evals_runner.py` | Test | Markers | Objective | |---|---|---| -| `RegistryRace.test_concurrent_identical_uploads_one_wins` | integration | Two threads upload the same content simultaneously. | +| `EvalsRunnerTests.test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | -| `RegistryRace.test_concurrent_same_name_different_content` | integration | Two threads upload different content under the same filename. | +| `EvalsRunnerTests.test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | + +| `EvalsRunnerTests.test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | -| `test_concurrent_identical_uploads_one_wins` | — | Two threads upload the same content simultaneously. | +| `EvalsRunnerTests.test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | -| `test_concurrent_same_name_different_content` | — | Two threads upload different content under the same filename. | +| `EvalsRunnerTests.test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | -## `tests/test_regression.py` +| `EvalsRunnerTests.test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | -| Test | Markers | Objective | -|---|---|---| -| `RegressionBomHandling.test_bom_csv_is_accepted` | — | — | +| `EvalsRunnerTests.test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | -| `RegressionCrlfLineEndings.test_crlf_csv_accepted` | — | — | +| `test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | -| `RegressionHeaderOnlyCsv.test_header_only_exits_cleanly` | — | — | +| `test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | -| `RegressionSingleColumnCsv.test_single_column_valid_rows_pass` | — | — | +| `test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | -| `RegressionNoTraceback.test_missing_env_vars_no_traceback` | — | — | +| `test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | -| `RegressionNoTraceback.test_nonexistent_file_no_traceback` | — | — | +| `test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | -| `RegressionNoTraceback.test_invalid_utf8_no_traceback` | — | — | +| `test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | -| `test_bom_csv_is_accepted` | — | — | +| `test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | -| `test_crlf_csv_accepted` | — | — | +## `tests/unit/test_evals_tier_e_x.py` -| `test_header_only_exits_cleanly` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `TierEDiffSignatures.test_identical_schemas_report_no_differences` | unit | — | -| `test_single_column_valid_rows_pass` | — | — | +| `TierEDiffSignatures.test_namespace_is_not_part_of_the_comparison` | unit | — | -| `test_missing_env_vars_no_traceback` | — | — | +| `TierEDiffSignatures.test_missing_table_is_caught` | unit | — | -| `test_nonexistent_file_no_traceback` | — | — | +| `TierEDiffSignatures.test_extra_table_is_caught` | unit | — | -| `test_invalid_utf8_no_traceback` | — | — | +| `TierEDiffSignatures.test_added_column_is_caught` | unit | — | -## `tests/test_review.py` +| `TierEDiffSignatures.test_changed_type_is_caught` | unit | — | -| Test | Markers | Objective | -|---|---|---| -| `Review.test_pages_through_every_row_with_validation` | integration | — | +| `TierEDiffSignatures.test_changed_nullability_is_caught` | unit | — | -| `Review.test_validates_rows_beyond_the_reported_error_cap` | integration | — | +| `TierEDiffSignatures.test_reordered_columns_are_caught_as_drift` | unit | — | -| `Review.test_row_decisions_record_actor_and_reason_and_the_latest_wins` | integration | — | +| `TierEDiffSignatures.test_env_targets_cover_all_four_environments` | unit | — | -| `Review.test_cell_edits_are_versions_over_the_unchanged_original` | integration | — | +| `TierEColumnSignatureQuery.test_query_is_sent_on_stdin_not_via_dash_c` | unit | — | -| `Review.test_refusals` | integration | — | +| `TierEColumnSignatureQuery.test_query_failure_is_reported_as_a_query_error` | unit | — | -| `Review.test_te_review_shows_the_database_reason_a_row_was_rejected` | integration | — | +| `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | -| `Review.test_changes_are_audited_and_insert_only` | integration | — | +| `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | -| `test_pages_through_every_row_with_validation` | — | — | +| `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | -| `test_validates_rows_beyond_the_reported_error_cap` | — | — | +| `TierXSqliteStructure.test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | unit | — | -| `test_row_decisions_record_actor_and_reason_and_the_latest_wins` | — | — | +| `TierXSqliteStructure.test_structure_is_read_back_from_executed_ddl` | unit | — | -| `test_cell_edits_are_versions_over_the_unchanged_original` | — | — | +| `TierXSqliteStructure.test_invalid_ddl_is_reported_rather_than_raised` | unit | — | -| `test_refusals` | — | — | +| `TierXSqliteStructure.test_sqlite_internal_tables_are_excluded` | unit | — | -| `test_te_review_shows_the_database_reason_a_row_was_rejected` | — | — | +| `TierXSqliteStructure.test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | unit | — | -| `test_changes_are_audited_and_insert_only` | — | — | +| `TierXSqliteStructure.test_known_missing_tables_are_genuinely_absent` | unit | — | -## `tests/test_review_cases.py` +| `TierXSqliteStructure.test_missing_plus_present_accounts_for_all_twelve_tables` | unit | — | -| Test | Markers | Objective | -|---|---|---| -| `ReviewCases.test_P02_paging_bounds` | integration | — | +| `TierRegistration.test_all_five_tiers_are_registered` | unit | — | -| `ReviewCases.test_P04_ragged_row_is_flagged` | integration | — | +| `test_identical_schemas_report_no_differences` | — | — | -| `ReviewCases.test_P05_a_failed_upload_can_be_reviewed` | integration | — | +| `test_namespace_is_not_part_of_the_comparison` | — | — | -| `ReviewCases.test_P08_an_edit_to_a_bad_value_is_flagged` | integration | — | +| `test_missing_table_is_caught` | — | — | -| `ReviewCases.test_P09_edit_by_column_index_and_to_an_empty_value` | integration | — | +| `test_extra_table_is_caught` | — | — | -| `ReviewCases.test_P10_reason_is_stored_trimmed` | integration | — | +| `test_added_column_is_caught` | — | — | -| `ReviewCases.test_P11_actor_is_the_signed_in_user` | integration | — | +| `test_changed_type_is_caught` | — | — | -| `ReviewCases.test_P12_every_role_can_view_rows_and_history` | integration | — | +| `test_changed_nullability_is_caught` | — | — | -| `ReviewCases.test_N05_row_numbers_outside_the_upload` | integration | — | +| `test_reordered_columns_are_caught_as_drift` | — | — | -| `ReviewCases.test_N06_column_index_out_of_range` | integration | — | +| `test_env_targets_cover_all_four_environments` | — | — | -| `ReviewCases.test_N07_unknown_upload_on_every_route` | integration | — | +| `test_query_is_sent_on_stdin_not_via_dash_c` | — | — | -| `ReviewCases.test_N08_N09_uploader_and_auditor_cannot_change` | integration | — | +| `test_query_failure_is_reported_as_a_query_error` | — | — | -| `ReviewCases.test_N10_an_invalid_session_is_refused` | integration | — | +| `test_genuinely_empty_schema_still_fails` | — | — | -| `ReviewCases.test_N11_another_tenant_cannot_see_or_change_the_upload` | integration | — | +| `test_placeholders_are_substituted` | — | — | -| `ReviewCases.test_N12_missing_or_wrong_body_fields` | integration | — | +| `test_every_known_table_has_a_substitution` | — | — | -| `ReviewCases.test_N13_review_tables_reject_update_delete_and_truncate` | integration | — | +| `test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | — | — | -| `ReviewCases.test_N14_database_rejects_a_blank_reason` | integration | — | +| `test_structure_is_read_back_from_executed_ddl` | — | — | -| `ReviewCases.test_N16_a_non_numeric_upload_id` | integration | — | +| `test_invalid_ddl_is_reported_rather_than_raised` | — | — | -| `test_P02_paging_bounds` | — | — | +| `test_sqlite_internal_tables_are_excluded` | — | — | -| `test_P04_ragged_row_is_flagged` | — | — | +| `test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | — | — | -| `test_P05_a_failed_upload_can_be_reviewed` | — | — | +| `test_known_missing_tables_are_genuinely_absent` | — | — | -| `test_P08_an_edit_to_a_bad_value_is_flagged` | — | — | +| `test_missing_plus_present_accounts_for_all_twelve_tables` | — | — | -| `test_P09_edit_by_column_index_and_to_an_empty_value` | — | — | +| `test_all_five_tiers_are_registered` | — | — | -| `test_P10_reason_is_stored_trimmed` | — | — | +## `tests/unit/test_preflight_ps1.py` -| `test_P11_actor_is_the_signed_in_user` | — | — | +| Test | Markers | Objective | +|---|---|---| +| `test_no_null_coalescing_operator` | regression, unit | `??` / `??=` is PowerShell 7+ only and fails to parse on 5.1. | -| `test_P12_every_role_can_view_rows_and_history` | — | — | +| `test_no_other_powershell7_only_syntax` | regression, unit | — | -| `test_N05_row_numbers_outside_the_upload` | — | — | +| `test_no_utf8_bom` | regression, unit | A BOM caused a noisy line-1 diff during the fix; keep the file BOM-free. | -| `test_N06_column_index_out_of_range` | — | — | +| `test_port_label_is_not_hardcoded` | regression, unit | The check label must follow the configured port (was fixed at 'port 5432'). | -| `test_N07_unknown_upload_on_every_route` | — | — | +| `test_connection_defaults_resolved_once_and_reused` | regression, unit | Defaults live in variables; connection commands must use them. | -| `test_N08_N09_uploader_and_auditor_cannot_change` | — | — | +| `test_default_port_and_host_unchanged` | regression, unit | — | -| `test_N10_an_invalid_session_is_refused` | — | — | +| `test_exit_code_contract_documented_and_implemented` | regression, unit | — | -| `test_N11_another_tenant_cannot_see_or_change_the_upload` | — | — | +| `test_parses_under_windows_powershell` | regression | Parse with the real PowerShell parser (5.1 on stock Windows). | -| `test_N12_missing_or_wrong_body_fields` | — | — | +| `test_runs_with_skippg_and_reports_summary` | regression | — | -| `test_N13_review_tables_reject_update_delete_and_truncate` | — | — | +| `test_default_port_label_when_pgport_unset` | regression | — | -| `test_N14_database_rejects_a_blank_reason` | — | — | +| `test_port_label_follows_pgport` | regression | — | -| `test_N16_a_non_numeric_upload_id` | — | — | +| `test_pg_checks_are_optional_never_fail` | regression | — | -## `tests/test_roles.py` +## `tests/unit/test_regression.py` | Test | Markers | Objective | |---|---|---| -| `MatrixIsDocumented.test_docs_table_matches_the_enforced_permissions` | unit | — | - -| `MatrixIsDocumented.test_every_action_has_a_route_and_only_known_roles` | unit | — | +| `RegressionBomHandling.test_bom_csv_is_accepted` | — | — | -| `RoleMatrixOverTheApi.test_every_role_against_every_route` | integration | — | +| `RegressionCrlfLineEndings.test_crlf_csv_accepted` | — | — | -| `RoleMatrixOverTheApi.test_no_session_requests_are_not_role_checked` | integration | — | +| `RegressionHeaderOnlyCsv.test_header_only_exits_cleanly` | — | — | -| `test_docs_table_matches_the_enforced_permissions` | — | — | +| `RegressionSingleColumnCsv.test_single_column_valid_rows_pass` | — | — | -| `test_every_action_has_a_route_and_only_known_roles` | — | — | +| `RegressionNoTraceback.test_missing_env_vars_no_traceback` | — | — | -| `test_every_role_against_every_route` | — | — | +| `RegressionNoTraceback.test_nonexistent_file_no_traceback` | — | — | -| `test_no_session_requests_are_not_role_checked` | — | — | +| `RegressionNoTraceback.test_invalid_utf8_no_traceback` | — | — | -## `tests/test_runtime_role.py` +| `test_bom_csv_is_accepted` | — | — | -| Test | Markers | Objective | -|---|---|---| -| `RuntimeRole.test_script_keeps_the_password_out_of_its_output` | integration | — | +| `test_crlf_csv_accepted` | — | — | -| `RuntimeRole.test_the_api_works_as_the_runtime_role` | integration | — | +| `test_header_only_exits_cleanly` | — | — | -| `RuntimeRole.test_runtime_role_cannot_change_or_unprotect_append_only_tables` | integration | — | +| `test_single_column_valid_rows_pass` | — | — | -| `test_script_keeps_the_password_out_of_its_output` | — | — | +| `test_missing_env_vars_no_traceback` | — | — | -| `test_the_api_works_as_the_runtime_role` | — | — | +| `test_nonexistent_file_no_traceback` | — | — | -| `test_runtime_role_cannot_change_or_unprotect_append_only_tables` | — | — | +| `test_invalid_utf8_no_traceback` | — | — | -## `tests/test_security.py` +## `tests/unit/test_security.py` | Test | Markers | Objective | |---|---|---| @@ -1473,7 +1609,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_env_sql_files_no_hardcoded_passwords` | — | — | -## `tests/test_service_loader.py` +## `tests/unit/test_service_loader.py` | Test | Markers | Objective | |---|---|---| @@ -1485,7 +1621,7 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_te_cleanup_prior_registry_entries_drops_dynamic_table` | — | — | -## `tests/test_snapshot.py` +## `tests/unit/test_snapshot.py` | Test | Markers | Objective | |---|---|---| @@ -1501,123 +1637,3 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_all_valid_no_skips` | — | — | -## `tests/test_te_loader_integration.py` - -| Test | Markers | Objective | -|---|---|---| -| `TeUploadOrganisations.test_valid_te_upload_returns_ok` | integration | A well-formed T&E CSV must return status=ok. | - -| `TeUploadOrganisations.test_inserted_rows_count_matches_data_rows` | integration | insertedRows must equal the number of data rows supplied. | - -| `TeUploadOrganisations.test_table_name_in_response_is_schema_qualified` | integration | tableName must be schema-qualified (e.g. te_dev.organisations). | - -| `TeUploadOrganisations.test_rows_are_queryable_after_upload` | integration | Inserted rows must appear in the T&E table row count. | - -| `TeUploadOrganisations.test_core_te_tables_undisturbed_after_upload` | integration | A T&E upload must not drop or alter the other 11 core tables. | - -| `TeUploadOrganisations.test_reupload_cleans_prior_registry_entry` | integration | Re-uploading the same file name must not duplicate registry entries. | - -| `TeUploadOrganisations.test_invalid_target_table_is_rejected` | integration | A targetTable not in TE_TABLES must return status=error. | - -| `TeUploadOrganisations.test_empty_file_is_rejected` | integration | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | - -| `TeUploadOrganisations.test_header_only_file_is_rejected` | integration | A header-only CSV (no data rows) must return status=invalid_structure. | - -| `TeUploadOrganisations.test_unknown_columns_are_rejected` | integration | Columns not in the target table must return status=error. | - -| `TeUploadOrganisations.test_missing_target_table_param_is_rejected` | integration | mode=te with no targetTable must return 422 from the API layer. | - -| `DynamicOverwriteOfTeOriginFile.test_dynamic_overwrite_of_te_origin_entry_succeeds` | integration | A dynamic-mode overwrite of a filename previously registered in | - -| `test_valid_te_upload_returns_ok` | — | A well-formed T&E CSV must return status=ok. | - -| `test_inserted_rows_count_matches_data_rows` | — | insertedRows must equal the number of data rows supplied. | - -| `test_table_name_in_response_is_schema_qualified` | — | tableName must be schema-qualified (e.g. te_dev.organisations). | - -| `test_rows_are_queryable_after_upload` | — | Inserted rows must appear in the T&E table row count. | - -| `test_core_te_tables_undisturbed_after_upload` | — | A T&E upload must not drop or alter the other 11 core tables. | - -| `test_reupload_cleans_prior_registry_entry` | — | Re-uploading the same file name must not duplicate registry entries. | - -| `test_invalid_target_table_is_rejected` | — | A targetTable not in TE_TABLES must return status=error. | - -| `test_empty_file_is_rejected` | — | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | - -| `test_header_only_file_is_rejected` | — | A header-only CSV (no data rows) must return status=invalid_structure. | - -| `test_unknown_columns_are_rejected` | — | Columns not in the target table must return status=error. | - -| `test_missing_target_table_param_is_rejected` | — | mode=te with no targetTable must return 422 from the API layer. | - -| `test_dynamic_overwrite_of_te_origin_entry_succeeds` | — | A dynamic-mode overwrite of a filename previously registered in | - -## `tests/test_te_reupload_registry.py` - -| Test | Markers | Objective | -|---|---|---| -| `TeReuploadRegistry.test_all_failed_reupload_keeps_the_earlier_registry_record` | integration | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | - -| `TeReuploadRegistry.test_reupload_that_inserts_rows_still_replaces_the_registry_record` | integration | A T&E re-upload that loads at least one row still replaces the registry record. | - -| `TeReuploadRegistry.test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | integration | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | - -| `test_all_failed_reupload_keeps_the_earlier_registry_record` | — | A T&E re-upload that fails every row leaves the earlier load's registry record alone. | - -| `test_reupload_that_inserts_rows_still_replaces_the_registry_record` | — | A T&E re-upload that loads at least one row still replaces the registry record. | - -| `test_all_failed_te_upload_keeps_a_prior_dynamic_table_of_the_same_name` | — | An all-failed T&E upload does not drop a dynamic table registered under the same filename (#170). | - -## `tests/test_tenant_isolation.py` - -| Test | Markers | Objective | -|---|---|---| -| `CrossTenantIsolation.test_upload_lands_in_the_tenants_own_schema` | integration | — | - -| `CrossTenantIsolation.test_other_tenant_cannot_list_read_or_delete` | integration | — | - -| `CrossTenantIsolation.test_requests_without_a_session_see_no_tenant_data` | integration | — | - -| `CrossTenantIsolation.test_duplicate_detection_is_per_tenant` | integration | — | - -| `CrossTenantIsolation.test_audit_log_is_per_tenant` | integration | — | - -| `CrossTenantIsolation.test_te_mode_loads_into_the_tenants_own_te_tables` | integration | — | - -| `CrossTenantIsolation.test_te_table_counts_are_per_tenant` | integration | — | - -| `CrossTenantIsolation.test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | integration | — | - -| `CrossTenantIsolation.test_invalid_session_is_401_not_a_fallback_to_default` | integration | — | - -| `CrossTenantIsolation.test_tenant_schema_is_at_head_with_its_own_registry` | integration | — | - -| `CrossTenantIsolation.test_concurrent_requests_never_see_another_tenants_schema` | integration | — | - -| `CrossTenantIsolation.test_provisioning_is_idempotent` | integration | — | - -| `test_upload_lands_in_the_tenants_own_schema` | — | — | - -| `test_other_tenant_cannot_list_read_or_delete` | — | — | - -| `test_requests_without_a_session_see_no_tenant_data` | — | — | - -| `test_duplicate_detection_is_per_tenant` | — | — | - -| `test_audit_log_is_per_tenant` | — | — | - -| `test_te_mode_loads_into_the_tenants_own_te_tables` | — | — | - -| `test_te_table_counts_are_per_tenant` | — | — | - -| `test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | — | — | - -| `test_invalid_session_is_401_not_a_fallback_to_default` | — | — | - -| `test_tenant_schema_is_at_head_with_its_own_registry` | — | — | - -| `test_concurrent_requests_never_see_another_tenants_schema` | — | — | - -| `test_provisioning_is_idempotent` | — | — | - diff --git a/pytest.ini b/pytest.ini index 49bc7cc..8ec98a4 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,6 +7,11 @@ markers = security: static security and credential pattern scans snapshot: golden-file output comparison tests parity: cross-environment row-count consistency tests + system: whole-component behaviour through its public surface (tests/system/) + sit: system integration — several components plus PostgreSQL (tests/system_integration/) + performance: timing and throughput budgets (tests/performance/) + accessibility: WCAG scans; needs Playwright browsers and a running frontend + uat: user-acceptance scenarios traced to an issue or user story (tests/uat/) addopts = -ra -q testpaths = tests diff --git a/requirements-dev.txt b/requirements-dev.txt index b69f383..0aeb6be 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,8 @@ PyYAML>=6.0.3 # it falls back to httpx with a deprecation warning, and raises if neither # is installed. Nothing else in this file or api/requirements.txt pulls it in. httpx2>=2.10.0 + +# #168: accessibility scans (tests/accessibility/). Importable without the +# browsers; the dedicated CI job runs `playwright install chromium` as well. +playwright>=1.49.0 +axe-playwright-python>=0.1.3 diff --git a/scripts/fix_inserted_rows_count.py b/scripts/fix_inserted_rows_count.py index 1b3363c..f618d38 100644 --- a/scripts/fix_inserted_rows_count.py +++ b/scripts/fix_inserted_rows_count.py @@ -24,7 +24,7 @@ Then verify:: - python -m pytest tests/test_issue_04_multi_file_upload.py + python -m pytest tests/uat/test_issue_04_multi_file_upload.py """ from __future__ import annotations @@ -100,8 +100,8 @@ def main() -> int: TARGET.write_text(updated, encoding="utf-8") print(" rewrote api/services/dynamic_loader.py") print("\nNow verify:") - print(" python -m pytest tests/test_issue_04_multi_file_upload.py") - print(" python -m pytest tests/test_issue_05_import_summary.py") + print(" python -m pytest tests/uat/test_issue_04_multi_file_upload.py") + print(" python -m pytest tests/uat/test_issue_05_import_summary.py") return 0 diff --git a/scripts/health_check.py b/scripts/health_check.py index e68d0d4..b76f31c 100644 --- a/scripts/health_check.py +++ b/scripts/health_check.py @@ -173,7 +173,7 @@ def check_test_infra() -> None: for label, path in files.items(): _check_file(label, path) - test_files = list((ROOT / "tests").glob("test_*.py")) + test_files = list((ROOT / "tests").rglob("test_*.py")) if test_files: _pass(f"python test files ({len(test_files)} found)") else: diff --git a/scripts/lint.sh b/scripts/lint.sh index 0d15b62..3b556f2 100644 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -27,30 +27,18 @@ PY_SRC=( scripts/create_tenant_admin.py ) -PY_TESTS=( - tests/test_csv_validator.py - tests/test_evals_runner.py - tests/test_api_unit.py - tests/test_api_integration.py - tests/test_issue_04_multi_file_upload.py - tests/test_issue_05_import_summary.py - tests/test_api_hardening.py - tests/test_evals_tier_e_x.py - tests/test_migrations.py - tests/test_import_counts.py - tests/test_accounts.py - tests/test_tenant_isolation.py - tests/test_roles.py - tests/test_oidc.py -) +# #167: every Python test file in the repo, so a new test cannot land unlinted. +# Collected dynamically — the category folders (tests/unit, tests/system, …) +# made the old hand-maintained list drift the moment a test was added. +mapfile -t PY_TESTS < <(find tests -name '*.py' | sort) # Collect scripts/*.py dynamically mapfile -t PY_SCRIPTS < <(ls scripts/*.py 2>/dev/null || true) -ALL_PY=("${PY_SRC[@]}" "${PY_TESTS[@]}" "${PY_SCRIPTS[@]}") +SRC_PY=("${PY_SRC[@]}" "${PY_SCRIPTS[@]}") -echo -e "\n${YELLOW}=== flake8 (style + logic) ===${NC}" -if python3 -m flake8 "${ALL_PY[@]}" \ +echo -e "\n${YELLOW}=== flake8 — source (${#SRC_PY[@]} files) ===${NC}" +if python3 -m flake8 "${SRC_PY[@]}" \ --max-line-length=120 \ --extend-ignore=E501,E221,E272; then echo -e "${GREEN}✓ flake8 clean${NC}" @@ -59,6 +47,20 @@ else exit 1 fi +# Tests use a deliberately compact table style — one-line class attribute +# blocks (`ENV = "dev"; DB = "te_mgmt_dev"`) and aligned continuations — that +# reads better as a fixture declaration than as PEP 8 prose. E702/E401/E128 are +# allowed there and nowhere else; every other check applies as above. +echo -e "\n${YELLOW}=== flake8 — tests (${#PY_TESTS[@]} files) ===${NC}" +if python3 -m flake8 "${PY_TESTS[@]}" \ + --max-line-length=120 \ + --extend-ignore=E501,E221,E272,E702,E401,E128; then + echo -e "${GREEN}✓ flake8 clean${NC}" +else + echo -e "${RED}✗ flake8 found issues in tests — fix before merging${NC}" + exit 1 +fi + echo -e "\n${YELLOW}=== bandit (security) ===${NC}" if python3 -m bandit "${PY_SRC[@]}" -ll -q 2>&1; then echo -e "${GREEN}✓ bandit clean${NC}" diff --git a/scripts/provision_full_test_env.sh b/scripts/provision_full_test_env.sh index 2a0f279..53a9fc5 100644 --- a/scripts/provision_full_test_env.sh +++ b/scripts/provision_full_test_env.sh @@ -20,7 +20,7 @@ # Note on passwords: the generated env_.sql keep the literal # '__INJECT_AT_DEPLOY__' placeholder from the template. It is a valid role # password for a local test instance and keeps -# tests/test_security.py::test_env_sql_files_no_hardcoded_passwords green. +# tests/unit/test_security.py::test_env_sql_files_no_hardcoded_passwords green. # Never use these files against a real environment. # ============================================================================= diff --git a/scripts/run_qa.ps1 b/scripts/run_qa.ps1 index 01c18c4..c1cf60d 100644 --- a/scripts/run_qa.ps1 +++ b/scripts/run_qa.ps1 @@ -42,8 +42,8 @@ switch ($Target) { "build/csv/validator.py", "evals/runner.py", "evals/gap_report.py", - "tests/test_csv_validator.py", - "tests/test_evals_runner.py" + "tests/unit/test_csv_validator.py", + "tests/unit/test_evals_runner.py" ) $scriptPy = Get-ChildItem scripts/*.py -ErrorAction SilentlyContinue | ForEach-Object { $_.FullName } diff --git a/scripts/select_tests.py b/scripts/select_tests.py index 60738a9..aec2210 100644 --- a/scripts/select_tests.py +++ b/scripts/select_tests.py @@ -32,7 +32,7 @@ ("build/environments/", "integration or e2e or parity", "env SQL changed → integration + parity"), ("build/schema/", "integration or e2e or parity", "schema changed → integration + parity"), ("tests/suites/", "integration", "SQL test suite changed → integration"), - ("tests/test_", "unit", "test file changed → unit"), + ("tests/", "unit", "test file changed → unit"), ("scripts/", "security", "scripts changed → security scan"), ("build/adapters/", "security", "adapter changed → security scan"), ("build/csv/loader_", "e2e", "loader changed → e2e"), diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..4d5797b --- /dev/null +++ b/tests/README.md @@ -0,0 +1,42 @@ +# Test categories + +Tests are grouped by **what they need to run**, not by what they cover. The +folder is the grouping; `tests/conftest.py` gives every test the marker of its +folder, so `pytest -m sit` and `pytest tests/system_integration` select the +same set. + +| Folder | Marker | Needs | Target | +|---|---|---|---| +| `unit/` | `unit` | nothing | `make test-unit` | +| `system/` | `system` | PostgreSQL | `make test-system` | +| `system_integration/` | `sit` | PostgreSQL, a deployed environment | `make test-sit` | +| `performance/` | `performance` | PostgreSQL | `make test-performance` | +| `accessibility/` | `accessibility` | Playwright browsers + the Vite dev server | `make test-accessibility` | +| `uat/` | `uat` | PostgreSQL | `make test-uat` | + +The older behaviour markers (`integration`, `e2e`, `regression`, `security`, +`snapshot`, `parity`) still apply and still drive CI's tier split. A test can +carry both: `tests/unit/test_regression.py` is `unit` *and* `regression`. + +## Which folder does a new test go in? + +- Imports a module and asserts on its return value → `unit/` +- Drives one component through its public surface (the HTTP API, the loader + CLI) and needs a database → `system/` +- Crosses components, or checks a deployed schema or environment → `system_integration/` +- Asserts a timing or throughput budget → `performance/` +- Asserts a WCAG rule against rendered pages → `accessibility/` +- Traces to an issue or a user story and reads as a user would describe it → `uat/` + +## No-skip policy + +An unmet prerequisite **fails** with remediation text; it does not skip. +`scripts/test_report.py --strict` enforces this in CI. The shared fixtures in +`tests/conftest.py` (`live_api`, `registry_cleanup`, `te_org_cleanup`, +`destructive_enabled`, `unique_tag`) already follow it — prefer them over a +per-module `setUpClass` that swallows a connection error. + +## Non-Python suites + +`suites/` and `framework/` hold the SQL assertion suites, run by +`evals/runner.py` (Tier S), not by pytest. diff --git a/tests/environments/__init__.py b/tests/accessibility/__init__.py similarity index 100% rename from tests/environments/__init__.py rename to tests/accessibility/__init__.py diff --git a/tests/accessibility/conftest.py b/tests/accessibility/conftest.py new file mode 100644 index 0000000..683a3aa --- /dev/null +++ b/tests/accessibility/conftest.py @@ -0,0 +1,109 @@ +"""Playwright + axe-core fixtures for the WCAG scans (#168). + +These tests drive a real browser against a running frontend, so they are the +only category with an out-of-process prerequisite. Per the repository's no-skip +policy a missing browser or an unreachable dev server FAILS with remediation +text rather than skipping. + +Run them with: + + cd csv-table-hub-main && bun run dev # or: npm run dev + python -m pytest tests/accessibility -v + +Point them somewhere else with A11Y_BASE_URL (e.g. a preview deployment). +""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request + +import pytest + +BASE_URL = os.environ.get("A11Y_BASE_URL", "http://localhost:5173").rstrip("/") + +# WCAG 2.1 level AA is the target (docs/STANDARDS_AUDIT.md). axe groups its +# rules by tag; these four cover A and AA for both 2.0 and 2.1. "best-practice" +# is deliberately excluded — it is advice, not the conformance target. +WCAG_21_AA_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"] + +_SERVER_HELP = ( + f"No frontend answered at {BASE_URL}.\n" + " cd csv-table-hub-main && bun install && bun run dev\n" + "Then re-run, or set A11Y_BASE_URL to a running instance." +) + +_BROWSER_HELP = ( + "Playwright could not launch Chromium.\n" + " python -m playwright install chromium\n" + "(the CI job does this; see .github/workflows/quality-gate.yml)" +) + + +@pytest.fixture(scope="session") +def base_url() -> str: + """The frontend under scan, verified reachable before any test runs.""" + try: + urllib.request.urlopen(BASE_URL, timeout=15) # nosec B310 — fixed http(s) literal + except urllib.error.HTTPError: + pass # a 4xx/5xx still means something is listening and serving + except Exception as exc: # noqa: BLE001 + pytest.fail(f"{type(exc).__name__}: {exc}\n{_SERVER_HELP}", pytrace=False) + return BASE_URL + + +@pytest.fixture(scope="session") +def browser(base_url: str): + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + try: + browser = p.chromium.launch(args=["--no-sandbox"]) + except Exception as exc: # noqa: BLE001 + pytest.fail(f"{type(exc).__name__}: {exc}\n{_BROWSER_HELP}", pytrace=False) + yield browser + browser.close() + + +# The frontend keeps its session in sessionStorage under this key +# (csv-table-hub-main/src/lib/session.ts) and `requireSession` redirects to +# /auth without one. Seeding it lets the scans reach the authenticated pages +# without a backend or a real account. +_SESSION = {"token": "a11y-scan", "expiresAt": "2099-01-01T00:00:00Z", + "username": "a11y", "tenant": "a11y", "role": "admin"} + + +def _empty_api(route): + """Answer every /api/** call with an empty list so pages render their + empty state instead of an error banner. The scan covers the frontend only.""" + route.fulfill(status=200, content_type="application/json", body="[]") + + +@pytest.fixture +def page(browser): + """A fresh page per test, at a desktop viewport. Signed out.""" + context = browser.new_context(viewport={"width": 1280, "height": 900}) + page = context.new_page() + yield page + context.close() + + +@pytest.fixture +def signed_in_page(browser): + """As `page`, but with a seeded session and a stubbed API.""" + context = browser.new_context(viewport={"width": 1280, "height": 900}) + context.add_init_script( + f"sessionStorage.setItem('te.session', {json.dumps(json.dumps(_SESSION))});") + context.route("**/api/**", _empty_api) + page = context.new_page() + yield page + context.close() + + +@pytest.fixture(scope="session") +def axe(): + from axe_playwright_python.sync_playwright import Axe + + return Axe() diff --git a/tests/accessibility/test_accessibility.py b/tests/accessibility/test_accessibility.py new file mode 100644 index 0000000..d9706e2 --- /dev/null +++ b/tests/accessibility/test_accessibility.py @@ -0,0 +1,72 @@ +"""WCAG 2.1 AA scans of the frontend's pages (#168). + +Known limitation, stated in #168: these scans fail against the frontend's +current, unfixed accessibility issues (sonner toast contrast, selects without +accessible names, the download menu's missing aria-labels, the upload progress +bar). That is the point of the job — the failures are the backlog, and they are +expected until those fixes land. The job is deliberately NOT in branch +protection yet. + +`/` and `/audit` are behind `requireSession`: signed out they redirect to +/auth, so scanning them signed out would just scan the sign-in page three times +and report a false green. The `signed_in_page` fixture seeds a session and stubs +the API, and every scan asserts it landed on the route it asked for. +""" + +from __future__ import annotations + +import pytest + +from .conftest import WCAG_21_AA_TAGS + +# (route, human name, needs a session) +PAGES = [ + ("/auth", "sign-in", False), + ("/", "upload", True), + ("/audit", "audit log", True), +] +IDS = [p[0] for p in PAGES] + + +def _open(page, base_url: str, route: str) -> None: + page.goto(f"{base_url}{route}", wait_until="networkidle") + # Guard against the false green: a redirect means we scanned a different page. + assert page.url.rstrip("/") == f"{base_url}{route}".rstrip("/"), ( + f"asked for {route} but landed on {page.url}") + + +@pytest.fixture(params=PAGES, ids=IDS) +def target(request, page, signed_in_page): + route, name, needs_session = request.param + return (signed_in_page if needs_session else page), route, name + + +def test_page_has_no_wcag_21_aa_violations(axe, target, base_url): + page, route, name = target + _open(page, base_url, route) + results = axe.run(page, options={ + "runOnly": {"type": "tag", "values": WCAG_21_AA_TAGS}, + "resultTypes": ["violations"], + }) + assert results.violations_count == 0, ( + f"{results.violations_count} WCAG 2.1 AA violation(s) on {name} ({route}):\n" + f"{results.generate_report()}" + ) + + +def test_page_has_one_main_landmark(target, base_url): + """Landmarks are how a screen-reader user skips the page chrome.""" + page, route, name = target + _open(page, base_url, route) + count = page.locator("main, [role=main]").count() + assert count == 1, f"{name} ({route}): found {count} main landmarks, expected 1" + + +def test_sign_in_form_fields_all_have_labels(axe, page, base_url): + """A form you cannot label is a form a screen-reader user cannot fill in.""" + _open(page, base_url, "/auth") + results = axe.run(page, options={ + "runOnly": {"type": "rule", "values": ["label", "form-field-multiple-labels"]}, + "resultTypes": ["violations"], + }) + assert results.violations_count == 0, results.generate_report() diff --git a/tests/conftest.py b/tests/conftest.py index 073df61..24cf864 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,3 +65,134 @@ def _restore_env(): os.environ.pop(k, None) else: os.environ[k] = v + + +# ── Test-group markers (#167) ───────────────────────────────────────────────── +# Tests are grouped by folder (see tests/README.md). Every test inherits the +# marker of the folder it lives in, so `pytest -m uat` and `pytest tests/uat` +# select the same set — without editing 36 files to add a second marker. +_GROUP_MARKERS = { + "unit": "unit", + "system": "system", + "system_integration": "sit", + "performance": "performance", + "accessibility": "accessibility", + "uat": "uat", +} +_TESTS_ROOT = Path(__file__).resolve().parent + + +def pytest_collection_modifyitems(config, items): + for item in items: + try: + rel = Path(str(item.fspath)).resolve().relative_to(_TESTS_ROOT) + except ValueError: + continue + marker = _GROUP_MARKERS.get(rel.parts[0]) if len(rel.parts) > 1 else None + if marker: + item.add_marker(getattr(pytest.mark, marker)) + + +# ── Live-database fixtures (system / SIT / performance / UAT) ───────────────── +_DB_HELP = ( + "Start PostgreSQL, set PGHOST/PGUSER/PGPASSWORD, point PGDATABASE at the " + "provisioned dev database and deploy it:\n" + " bash scripts/provision_full_test_env.sh\n" + " export PGDATABASE=te_mgmt_dev" +) + + +@pytest.fixture(scope="module") +def live_api(): + """A TestClient with the FastAPI lifespan running (pool + bootstrap). + + Per the repository's no-skip policy an unreachable database FAILS with + remediation text rather than skipping. + """ + from fastapi.testclient import TestClient + + ctx = TestClient(app) + try: + client = ctx.__enter__() + except Exception as exc: # noqa: BLE001 + pytest.fail( + "Could not start the API — database connection failed.\n" + f" {type(exc).__name__}: {str(exc).strip().splitlines()[0]}\n{_DB_HELP}", + pytrace=False, + ) + if client.get("/api/health").json().get("status") != "ok": + ctx.__exit__(None, None, None) + pytest.fail(f"API reports the database is unreachable.\n{_DB_HELP}", pytrace=False) + yield client + ctx.__exit__(None, None, None) + + +@pytest.fixture +def unique_tag(request): + """Short tag unique to this test invocation — keeps content hashes from + colliding with rows left behind by earlier runs.""" + import uuid + + return f"{request.node.name[:40]}_{uuid.uuid4().hex[:8]}" + + +@pytest.fixture +def registry_cleanup(live_api): + """Collects uploaded file names and removes their registry rows (and, for + dynamic uploads, their csv_ tables) after the test, directly via SQL + so cleanup works even while API_ALLOW_DESTRUCTIVE is off.""" + names: list[str] = [] + yield names + if not names: + return + from psycopg2 import sql + + from api.db import Conn + from api.tenancy import uploads_schema + + schema = uploads_schema() + with Conn() as conn: + with conn.cursor() as cur: + cur.execute( + sql.SQL("SELECT id, table_name, mode FROM {}.csv_files WHERE file_name = ANY(%s)") + .format(sql.Identifier(schema)), + (names,), + ) + for file_id, table_name, mode in cur.fetchall(): + if mode == "dynamic" and table_name.startswith("csv_"): + cur.execute(sql.SQL("DROP TABLE IF EXISTS {}.{}").format( + sql.Identifier(schema), sql.Identifier(table_name))) + cur.execute(sql.SQL("DELETE FROM {}.csv_files WHERE id = %s") + .format(sql.Identifier(schema)), (file_id,)) + conn.commit() + + +@pytest.fixture +def destructive_enabled(): + """Temporarily enable DELETE /api/csv/files/{id} (API_ALLOW_DESTRUCTIVE).""" + original = _settings.allow_destructive + _settings.allow_destructive = True + yield + _settings.allow_destructive = original + + +@pytest.fixture +def te_org_cleanup(live_api): + """Collects organisation names inserted into te_dev.organisations by a + T&E-mode test and deletes them afterwards so seed data stays intact.""" + names: list[str] = [] + yield names + if not names: + return + from psycopg2 import sql + + from api.db import Conn + + with Conn() as conn: + with conn.cursor() as cur: + cur.execute( + sql.SQL("DELETE FROM {}.organisations WHERE name = ANY(%s)") + .format(sql.Identifier(_settings.TE_SCHEMA)), + (names,), + ) + conn.commit() diff --git a/tests/performance/__init__.py b/tests/performance/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/system/__init__.py b/tests/system/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_api_coverage.py b/tests/system/test_api_coverage.py similarity index 99% rename from tests/test_api_coverage.py rename to tests/system/test_api_coverage.py index 1ac0e1c..3a56105 100644 --- a/tests/test_api_coverage.py +++ b/tests/system/test_api_coverage.py @@ -280,7 +280,6 @@ def test_deleted_file_no_longer_listed(self): self.assertNotIn(str(file_id), ids) - # --------------------------------------------------------------------------- # BUG-041: GET /api/audit/log (integration) # --------------------------------------------------------------------------- @@ -419,7 +418,6 @@ def test_te_upload_registers_in_csv_files(self): self.assertTrue(match, "T&E upload should appear in csv_files list") self.assertEqual(match[0]["mode"], "te") - def test_enum_constraint_violation_reported_per_row(self): """An invalid enum value must produce a per-row error, not a 500.""" csv = self._org_csv(f"{self.tag}_enum", org_type="INVALID_ENUM") diff --git a/tests/test_csv_loader_arbitrary_shapes.py b/tests/system/test_csv_loader_arbitrary_shapes.py similarity index 98% rename from tests/test_csv_loader_arbitrary_shapes.py rename to tests/system/test_csv_loader_arbitrary_shapes.py index d079a36..e3e6c0b 100644 --- a/tests/test_csv_loader_arbitrary_shapes.py +++ b/tests/system/test_csv_loader_arbitrary_shapes.py @@ -21,7 +21,7 @@ pytestmark = pytest.mark.integration -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] LOADER = ROOT / "build" / "csv_loader.sh" UTILISE = ROOT / "build" / "csv_utilise.sh" BUILD_DIR = ROOT / "build" diff --git a/tests/test_e2e_pipeline.py b/tests/system/test_e2e_pipeline.py similarity index 98% rename from tests/test_e2e_pipeline.py rename to tests/system/test_e2e_pipeline.py index 3134bec..d6fd899 100644 --- a/tests/test_e2e_pipeline.py +++ b/tests/system/test_e2e_pipeline.py @@ -19,7 +19,7 @@ pytestmark = pytest.mark.e2e -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] VALIDATOR = ROOT / "build" / "csv" / "validator.py" RUNNER = ROOT / "evals" / "runner.py" @@ -83,7 +83,7 @@ def test_happy_path_two_valid_rows(self): self.assertIn("2,Bob", valid_lines) # skip.csv should contain only the header (no skips) if skip_csv.exists(): - skip_lines = [l for l in skip_csv.read_text(encoding="utf-8").splitlines() if l.strip()] + skip_lines = [ln for ln in skip_csv.read_text(encoding="utf-8").splitlines() if ln.strip()] self.assertLessEqual(len(skip_lines), 1) def test_mixed_rows_splits_correctly(self): diff --git a/tests/system_integration/__init__.py b/tests/system_integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/system_integration/environments/__init__.py b/tests/system_integration/environments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/environments/_contract.py b/tests/system_integration/environments/_contract.py similarity index 100% rename from tests/environments/_contract.py rename to tests/system_integration/environments/_contract.py diff --git a/tests/environments/test_env_dev.py b/tests/system_integration/environments/test_env_dev.py similarity index 79% rename from tests/environments/test_env_dev.py rename to tests/system_integration/environments/test_env_dev.py index ad541da..946cbe5 100644 --- a/tests/environments/test_env_dev.py +++ b/tests/system_integration/environments/test_env_dev.py @@ -1,7 +1,7 @@ """Contract tests for the dev environment.""" import unittest import pytest -from tests.environments._contract import EnvironmentContract +from tests.system_integration.environments._contract import EnvironmentContract pytestmark = [pytest.mark.integration, pytest.mark.parity] diff --git a/tests/environments/test_env_prod.py b/tests/system_integration/environments/test_env_prod.py similarity index 80% rename from tests/environments/test_env_prod.py rename to tests/system_integration/environments/test_env_prod.py index 71f25e9..d285737 100644 --- a/tests/environments/test_env_prod.py +++ b/tests/system_integration/environments/test_env_prod.py @@ -1,7 +1,7 @@ """Contract tests for the prod environment.""" import unittest import pytest -from tests.environments._contract import EnvironmentContract +from tests.system_integration.environments._contract import EnvironmentContract pytestmark = [pytest.mark.integration, pytest.mark.parity] diff --git a/tests/environments/test_env_staging.py b/tests/system_integration/environments/test_env_staging.py similarity index 80% rename from tests/environments/test_env_staging.py rename to tests/system_integration/environments/test_env_staging.py index aac766a..e8a0bbb 100644 --- a/tests/environments/test_env_staging.py +++ b/tests/system_integration/environments/test_env_staging.py @@ -1,7 +1,7 @@ """Contract tests for the staging environment.""" import unittest import pytest -from tests.environments._contract import EnvironmentContract +from tests.system_integration.environments._contract import EnvironmentContract pytestmark = [pytest.mark.integration, pytest.mark.parity] diff --git a/tests/environments/test_env_test.py b/tests/system_integration/environments/test_env_test.py similarity index 80% rename from tests/environments/test_env_test.py rename to tests/system_integration/environments/test_env_test.py index 38efb30..3a7110a 100644 --- a/tests/environments/test_env_test.py +++ b/tests/system_integration/environments/test_env_test.py @@ -1,7 +1,7 @@ """Contract tests for the test environment.""" import unittest import pytest -from tests.environments._contract import EnvironmentContract +from tests.system_integration.environments._contract import EnvironmentContract pytestmark = [pytest.mark.integration, pytest.mark.parity] diff --git a/tests/test_accounts.py b/tests/system_integration/test_accounts.py similarity index 100% rename from tests/test_accounts.py rename to tests/system_integration/test_accounts.py diff --git a/tests/test_api_integration.py b/tests/system_integration/test_api_integration.py similarity index 100% rename from tests/test_api_integration.py rename to tests/system_integration/test_api_integration.py diff --git a/tests/test_audit_append_only.py b/tests/system_integration/test_audit_append_only.py similarity index 98% rename from tests/test_audit_append_only.py rename to tests/system_integration/test_audit_append_only.py index de7e957..34148b8 100644 --- a/tests/test_audit_append_only.py +++ b/tests/system_integration/test_audit_append_only.py @@ -19,7 +19,7 @@ from api.config import settings from api.services.audit import acting_as, verify_chain, write_audit -_ALEMBIC_INI = str(Path(__file__).resolve().parent.parent / "alembic.ini") +_ALEMBIC_INI = str(Path(__file__).resolve().parents[2] / "alembic.ini") def _connect(): diff --git a/tests/test_failed_uploads.py b/tests/system_integration/test_failed_uploads.py similarity index 100% rename from tests/test_failed_uploads.py rename to tests/system_integration/test_failed_uploads.py diff --git a/tests/test_header_mapping.py b/tests/system_integration/test_header_mapping.py similarity index 100% rename from tests/test_header_mapping.py rename to tests/system_integration/test_header_mapping.py diff --git a/tests/test_import_counts.py b/tests/system_integration/test_import_counts.py similarity index 100% rename from tests/test_import_counts.py rename to tests/system_integration/test_import_counts.py diff --git a/tests/test_migrations.py b/tests/system_integration/test_migrations.py similarity index 99% rename from tests/test_migrations.py rename to tests/system_integration/test_migrations.py index ad818b4..d372384 100644 --- a/tests/test_migrations.py +++ b/tests/system_integration/test_migrations.py @@ -28,7 +28,7 @@ _HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/" "PGPASSWORD/PGDATABASE). See scripts/start-api.ps1.") -_ALEMBIC_INI = str(Path(__file__).resolve().parent.parent / "alembic.ini") +_ALEMBIC_INI = str(Path(__file__).resolve().parents[2] / "alembic.ini") def _connect(): diff --git a/tests/test_oidc.py b/tests/system_integration/test_oidc.py similarity index 100% rename from tests/test_oidc.py rename to tests/system_integration/test_oidc.py diff --git a/tests/test_original_bytes.py b/tests/system_integration/test_original_bytes.py similarity index 100% rename from tests/test_original_bytes.py rename to tests/system_integration/test_original_bytes.py diff --git a/tests/test_parity.py b/tests/system_integration/test_parity.py similarity index 99% rename from tests/test_parity.py rename to tests/system_integration/test_parity.py index dfe5943..55d7349 100644 --- a/tests/test_parity.py +++ b/tests/system_integration/test_parity.py @@ -17,7 +17,7 @@ pytestmark = [pytest.mark.integration, pytest.mark.parity] -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] # Expected minimum seed row counts for the dev environment (from seed data). # Adjust these values if the seed SQL is intentionally changed. diff --git a/tests/test_password_reset.py b/tests/system_integration/test_password_reset.py similarity index 98% rename from tests/test_password_reset.py rename to tests/system_integration/test_password_reset.py index 2ca9c14..983751b 100644 --- a/tests/test_password_reset.py +++ b/tests/system_integration/test_password_reset.py @@ -18,10 +18,10 @@ from api.db import Conn from api.services import accounts from api.tenancy import migrate_tenant, schema_for -from tests.test_accounts import _PW, _Accounts +from tests.system_integration.test_accounts import _PW, _Accounts _NEW = "a brand new passphrase" -_ROOT = Path(__file__).resolve().parent.parent +_ROOT = Path(__file__).resolve().parents[2] def _bearer(token: str) -> dict: diff --git a/tests/test_preserved_uploads.py b/tests/system_integration/test_preserved_uploads.py similarity index 100% rename from tests/test_preserved_uploads.py rename to tests/system_integration/test_preserved_uploads.py diff --git a/tests/test_ragged_rows.py b/tests/system_integration/test_ragged_rows.py similarity index 62% rename from tests/test_ragged_rows.py rename to tests/system_integration/test_ragged_rows.py index 28bedaa..78cb325 100644 --- a/tests/test_ragged_rows.py +++ b/tests/system_integration/test_ragged_rows.py @@ -95,3 +95,47 @@ def test_multiple_ragged_rows_each_get_a_reason(self): self.assertIn("rowNumber", err, f"Missing rowNumber: {err}") self.assertIn("reason", err, f"Missing reason: {err}") self.assertTrue(err["reason"], f"Empty reason: {err}") + + # ── #192: what the tests above don't check — the reason is *stored* with the upload + # (GET /api/csv/uploads/{id}), not just returned, and T&E mode accounts the same way. + def _assert_accounted(self, res: dict, ragged_rows: dict[int, str]): + """total = inserted + duplicates + failed, and every ragged row has a + reason both in the response and stored with the upload.""" + self.assertEqual(res["status"], "ok") + self.assertEqual( + res["totalRows"], + res["insertedRows"] + res["duplicateRowsSkipped"] + res["failedRows"], + f"rows went missing: {res}") + self.assertEqual(res["failedRows"], len(ragged_rows)) + + def by_row(errors): + return {e["rowNumber"]: e for e in errors} + + returned = by_row(res["rowErrors"]) + self.assertEqual(set(returned), set(ragged_rows)) + stored_detail = self.client.get(f"/api/csv/uploads/{res['uploadId']}").json() + stored = by_row(stored_detail["rowErrors"]) + for row_number, lost in ragged_rows.items(): + for where, err in (("response", returned[row_number]), ("stored", stored[row_number])): + self.assertEqual(err["value"], lost, f"row {row_number} {where}") + self.assertIn("would be lost", err["reason"], f"row {row_number} {where}") + self.assertEqual(stored_detail["failedRows"], len(ragged_rows)) + + def test_dynamic_mode_accounts_for_every_row_with_a_stored_reason(self): + t = self.tag + res = self._upload( + f"a,b\n1,{t}\n1,{t}\n2,{t},LOST1\n3,{t}\n4,{t},LOST2,MORE\n") + self.assertEqual((res["insertedRows"], res["duplicateRowsSkipped"], res["failedRows"]), (2, 1, 2)) + self._assert_accounted(res, {3: "LOST1", 5: "LOST2"}) + + def test_te_mode_accounts_for_every_row_with_a_stored_reason(self): + t = self.tag + head = "name,org_type,country\n" + res = self._upload( + head + f"RAG_{t}_1,government,AU\n" + f"RAG_{t}_2,government,AU,LOST1\n" + f"RAG_{t}_3,government,AU\n" + f"RAG_{t}_4,government,AU,LOST2,MORE\n", + mode="te", targetTable="organisations") + self.assertEqual((res["insertedRows"], res["failedRows"]), (2, 2)) + self._assert_accounted(res, {2: "LOST1", 4: "LOST2"}) diff --git a/tests/system_integration/test_registry_race.py b/tests/system_integration/test_registry_race.py new file mode 100644 index 0000000..0fe0674 --- /dev/null +++ b/tests/system_integration/test_registry_race.py @@ -0,0 +1,163 @@ +"""Concurrent uploads must leave one registry row and no partial state (#191, "F1"). + +(Not #178, which is also labelled F1 and is a different defect.) + +The registry invariant is enforced by the database — migration 0002 adds +UNIQUE (file_hash) and UNIQUE (file_name) on csv_files — but the check-then-insert +in the upload path was not covered by any test. This drives the loader from +several threads released together by a barrier, so the requests genuinely overlap, +and repeats the race over several rounds because which interleaving wins is up to +the scheduler. Whatever the interleaving, the invariants below must hold. + +Mutation check (from #191): drop the file_hash constraint and +test_identical_content_under_different_names_registers_once fails (10 of 10 runs +at ROUNDS = 40) — + + ALTER TABLE csv_uploads.csv_files DROP CONSTRAINT csv_files_file_hash_key; + +Restore it afterwards with `ADD CONSTRAINT csv_files_file_hash_key UNIQUE (file_hash)` +(delete any duplicate rows the mutated run left first). + +Only that test detects it, and that is by design: when racers share a file *name* +as well as content, UNIQUE (file_name) and the csv_ table name also stop the +duplicate, so the hash constraint is not the only guard in that scenario. The +different-names scenario isolates it. +""" + +from __future__ import annotations + +import hashlib +import threading +from concurrent.futures import ThreadPoolExecutor + +import pytest +from psycopg2 import sql + +from api.db import Conn +from api.services.dynamic_loader import upload_dynamic +from api.tenancy import uploads_schema + +pytestmark = pytest.mark.integration + +ROUNDS = 40 +RACERS = 4 + + +def _race(uploads: list[tuple[str, str]]) -> list[dict]: + """Run each (file_name, content) upload at once; return the results in order.""" + gate = threading.Barrier(len(uploads)) + + def go(item): + gate.wait(timeout=30) + return upload_dynamic(item[0], item[1], None, False) + + with ThreadPoolExecutor(max_workers=len(uploads)) as pool: + return list(pool.map(go, uploads)) + + +def _registry(names: list[str], hashes: list[str]) -> list[tuple]: + with Conn() as conn, conn.cursor() as cur: + cur.execute(sql.SQL( + "SELECT file_name, file_hash, table_name, row_count FROM {}.csv_files " + "WHERE file_name = ANY(%s) OR file_hash = ANY(%s)").format(sql.Identifier(uploads_schema())), + (names, hashes)) + return cur.fetchall() + + +def _tables(prefixes: list[str]) -> list[str]: + with Conn() as conn, conn.cursor() as cur: + cur.execute("SELECT table_name FROM information_schema.tables " + "WHERE table_schema = %s AND table_name = ANY(%s)", + (uploads_schema(), prefixes)) + return sorted(r[0] for r in cur.fetchall()) + + +def _table_rows(table: str) -> int: + with Conn() as conn, conn.cursor() as cur: + cur.execute(sql.SQL("SELECT count(*) FROM {}.{}").format( + sql.Identifier(uploads_schema()), sql.Identifier(table))) + return cur.fetchone()[0] + + +def _assert_rejection_is_attributable(res: dict, winner_name: str): + """The loser is told it lost, and to whom — not a bare error or a 500.""" + assert res["status"] == "duplicate_file", res + assert res["existingFileName"] == winner_name, res + assert res["reason"] in ("content", "name"), res + + +def test_identical_file_uploaded_concurrently_registers_once(live_api, registry_cleanup, unique_tag): + for n in range(ROUNDS): + name = f"race_same_{unique_tag}_{n}.csv" + content = f"a,b\n1,{unique_tag}_{n}\n2,{unique_tag}_{n}\n" + digest = hashlib.sha256(content.encode()).hexdigest() + registry_cleanup.append(name) + + results = _race([(name, content)] * RACERS) + + ok = [r for r in results if r["status"] == "ok"] + rejected = [r for r in results if r["status"] != "ok"] + assert len(ok) == 1, f"round {n}: expected exactly one winner, got {[r['status'] for r in results]}" + assert len(rejected) == RACERS - 1 + for r in rejected: + _assert_rejection_is_attributable(r, name) + + # One registry row; one table; and the table holds the file once, not twice. + rows = _registry([name], [digest]) + assert len(rows) == 1, f"round {n}: registry rows {rows}" + assert _tables([f"csv_{digest[:16]}"]) == [f"csv_{digest[:16]}"] + assert _table_rows(f"csv_{digest[:16]}") == 2, f"round {n}: rows duplicated or lost" + + +def test_identical_content_under_different_names_registers_once(live_api, registry_cleanup, unique_tag): + """The case where file_hash, not file_name, is the guard: nothing about the + names clashes, so only the content identity can stop a second registration.""" + for n in range(ROUNDS): + content = f"a,b\n1,{unique_tag}_{n}\n2,{unique_tag}_{n}\n" + digest = hashlib.sha256(content.encode()).hexdigest() + names = [f"race_hash_{unique_tag}_{n}_{i}.csv" for i in range(RACERS)] + registry_cleanup.extend(names) + + results = _race([(nm, content) for nm in names]) + + ok = [(i, r) for i, r in enumerate(results) if r["status"] == "ok"] + assert len(ok) == 1, f"round {n}: expected exactly one winner, got {[r['status'] for r in results]}" + winner_name = names[ok[0][0]] + for i, r in enumerate(results): + if i != ok[0][0]: + _assert_rejection_is_attributable(r, winner_name) + assert r["reason"] == "content", r + + rows = _registry(names, [digest]) + assert len(rows) == 1, f"round {n}: registry rows {rows}" + assert rows[0][0] == winner_name + assert _table_rows(f"csv_{digest[:16]}") == 2, f"round {n}: rows duplicated or lost" + + +def test_same_name_different_content_uploaded_concurrently_registers_once(live_api, registry_cleanup, unique_tag): + for n in range(ROUNDS): + name = f"race_name_{unique_tag}_{n}.csv" + uploads = [(name, f"a,b\n{i},{unique_tag}_{n}_{i}\n") for i in range(RACERS)] + digests = [hashlib.sha256(c.encode()).hexdigest() for _, c in uploads] + registry_cleanup.append(name) + + results = _race(uploads) + + ok = [(i, r) for i, r in enumerate(results) if r["status"] == "ok"] + assert len(ok) == 1, f"round {n}: expected exactly one winner, got {[r['status'] for r in results]}" + winner = ok[0][0] + for i, r in enumerate(results): + if i != winner: + _assert_rejection_is_attributable(r, name) + assert r["reason"] == "name", r + + rows = _registry([name], digests) + assert len(rows) == 1, f"round {n}: registry rows {rows}" + assert rows[0][1] == digests[winner], "registry holds a different upload than the winner" + + # No partial state: only the winner's table exists. A loser's table + # (created before its registry insert failed) must have rolled back. + expected = [f"csv_{digests[winner][:16]}"] + present = _tables([f"csv_{d[:16]}" for d in digests]) + assert present == expected, f"round {n}: orphan tables {sorted(set(present) - set(expected))}" + assert _table_rows(expected[0]) == 1 diff --git a/tests/test_review.py b/tests/system_integration/test_review.py similarity index 100% rename from tests/test_review.py rename to tests/system_integration/test_review.py diff --git a/tests/test_review_cases.py b/tests/system_integration/test_review_cases.py similarity index 100% rename from tests/test_review_cases.py rename to tests/system_integration/test_review_cases.py diff --git a/tests/test_roles.py b/tests/system_integration/test_roles.py similarity index 98% rename from tests/test_roles.py rename to tests/system_integration/test_roles.py index 55d2775..79c5a6e 100644 --- a/tests/test_roles.py +++ b/tests/system_integration/test_roles.py @@ -24,7 +24,7 @@ from api.tenancy import migrate_tenant, schema_for _PW = "correct horse battery staple" -_DOC = Path(__file__).resolve().parent.parent / "docs" / "ROLES.md" +_DOC = Path(__file__).resolve().parents[2] / "docs" / "ROLES.md" # action -> (method, path); {table} and {id} are filled per run. _ROUTES = { "preview": ("POST", "/api/csv/preview"), diff --git a/tests/test_runtime_role.py b/tests/system_integration/test_runtime_role.py similarity index 99% rename from tests/test_runtime_role.py rename to tests/system_integration/test_runtime_role.py index 0ac0858..8b7c7b3 100644 --- a/tests/test_runtime_role.py +++ b/tests/system_integration/test_runtime_role.py @@ -25,7 +25,7 @@ from api.db import Conn from api.main import app -_ROOT = Path(__file__).resolve().parent.parent +_ROOT = Path(__file__).resolve().parents[2] def _owner(): diff --git a/tests/test_te_loader_integration.py b/tests/system_integration/test_te_loader_integration.py similarity index 99% rename from tests/test_te_loader_integration.py rename to tests/system_integration/test_te_loader_integration.py index 4df7e05..7282acf 100644 --- a/tests/test_te_loader_integration.py +++ b/tests/system_integration/test_te_loader_integration.py @@ -13,7 +13,7 @@ Run:: - pytest tests/test_te_loader_integration.py + pytest tests/system_integration/test_te_loader_integration.py Requires a provisioned dev environment: bash scripts/provision_full_test_env.sh diff --git a/tests/test_te_reupload_registry.py b/tests/system_integration/test_te_reupload_registry.py similarity index 100% rename from tests/test_te_reupload_registry.py rename to tests/system_integration/test_te_reupload_registry.py diff --git a/tests/test_tenant_isolation.py b/tests/system_integration/test_tenant_isolation.py similarity index 100% rename from tests/test_tenant_isolation.py rename to tests/system_integration/test_tenant_isolation.py diff --git a/tests/test_registry_race.py b/tests/test_registry_race.py deleted file mode 100644 index a40b8e2..0000000 --- a/tests/test_registry_race.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Concurrent uploads of the same file must not create duplicate registry rows (#191). - -The UNIQUE constraints on csv_files(file_hash) and csv_files(file_name) -settle the race at the database level. This test verifies that two -simultaneous uploads of the same content yield one success and one -graceful rejection — not a duplicate row or partial state. -""" - -from __future__ import annotations - -import os -import unittest -import uuid -from concurrent.futures import ThreadPoolExecutor, as_completed - -import pytest -from fastapi.testclient import TestClient - -from api.main import app - -_HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/" - "PGPASSWORD/PGDATABASE). See scripts/start-api.ps1.") - - -@pytest.mark.integration -class RegistryRace(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.ctx = TestClient(app) - try: - cls.client = cls.ctx.__enter__() - except Exception as exc: # noqa: BLE001 - raise AssertionError( - f"Could not start the API: {type(exc).__name__}: {exc}\n{_HELP}" - ) from None - - @classmethod - def tearDownClass(cls): - cls.ctx.__exit__(None, None, None) - - def setUp(self): - self.tag = f"{os.getpid()}_{uuid.uuid4().hex[:8]}" - - def _upload(self, file_name: str, content: str) -> dict: - body = {"fileName": file_name, "content": content} - return self.client.post("/api/csv/upload", json=body).json() - - def test_concurrent_identical_uploads_one_wins(self): - """Two threads upload the same content simultaneously. - - One should succeed; the other should fail gracefully (duplicate - hash detected). The registry must contain exactly one row. - """ - fname = f"race_same_{self.tag}.csv" - content = f"a,b\n1,{self.tag}\n" - - results: list[dict] = [] - with ThreadPoolExecutor(max_workers=2) as pool: - futures = [pool.submit(self._upload, fname, content) for _ in range(2)] - for f in as_completed(futures): - results.append(f.result()) - - statuses = [r.get("status") for r in results] - ok_count = statuses.count("ok") - self.assertGreaterEqual(ok_count, 1, f"At least one upload must succeed: {results}") - self.assertLessEqual(ok_count + statuses.count("error"), 2) - - files_resp = self.client.get("/api/csv/files").json() - matches = [f for f in files_resp if f.get("file_name") == fname] - self.assertEqual(len(matches), 1, f"Registry must have exactly one row: {matches}") - - def test_concurrent_same_name_different_content(self): - """Two threads upload different content under the same filename. - - The UNIQUE(file_name) constraint means only one can register. The - other must fail or be treated as a reupload (depending on the - code path), but not leave a second registry row. - """ - fname = f"race_name_{self.tag}.csv" - content_a = f"a,b\nA,{self.tag}\n" - content_b = f"x,y\nB,{self.tag}\n" - - results: list[dict] = [] - with ThreadPoolExecutor(max_workers=2) as pool: - fa = pool.submit(self._upload, fname, content_a) - fb = pool.submit(self._upload, fname, content_b) - for f in as_completed([fa, fb]): - results.append(f.result()) - - files_resp = self.client.get("/api/csv/files").json() - matches = [f for f in files_resp if f.get("file_name") == fname] - self.assertLessEqual( - len(matches), 1, - f"At most one registry row per filename: {matches}", - ) diff --git a/tests/uat/__init__.py b/tests/uat/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_issue_04_multi_file_upload.py b/tests/uat/test_issue_04_multi_file_upload.py similarity index 99% rename from tests/test_issue_04_multi_file_upload.py rename to tests/uat/test_issue_04_multi_file_upload.py index 4982e9c..19eb2a0 100644 --- a/tests/test_issue_04_multi_file_upload.py +++ b/tests/uat/test_issue_04_multi_file_upload.py @@ -17,7 +17,7 @@ Run:: - python -m pytest tests/test_issue_04_multi_file_upload.py + python -m pytest tests/uat/test_issue_04_multi_file_upload.py Requires a reachable database; per the repository's no-skip policy a missing prerequisite FAILS with remediation text rather than skipping. diff --git a/tests/test_issue_05_import_summary.py b/tests/uat/test_issue_05_import_summary.py similarity index 99% rename from tests/test_issue_05_import_summary.py rename to tests/uat/test_issue_05_import_summary.py index a0ecc5b..dd11692 100644 --- a/tests/test_issue_05_import_summary.py +++ b/tests/uat/test_issue_05_import_summary.py @@ -23,7 +23,7 @@ Run:: - python -m pytest tests/test_issue_05_import_summary.py + python -m pytest tests/uat/test_issue_05_import_summary.py Requires a reachable database; per the repository's no-skip policy a missing prerequisite FAILS with remediation text rather than skipping. diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/snapshots/basic_expected_skip.csv b/tests/unit/snapshots/basic_expected_skip.csv similarity index 100% rename from tests/snapshots/basic_expected_skip.csv rename to tests/unit/snapshots/basic_expected_skip.csv diff --git a/tests/snapshots/basic_expected_valid.csv b/tests/unit/snapshots/basic_expected_valid.csv similarity index 100% rename from tests/snapshots/basic_expected_valid.csv rename to tests/unit/snapshots/basic_expected_valid.csv diff --git a/tests/snapshots/basic_input.csv b/tests/unit/snapshots/basic_input.csv similarity index 100% rename from tests/snapshots/basic_input.csv rename to tests/unit/snapshots/basic_input.csv diff --git a/tests/test_api_hardening.py b/tests/unit/test_api_hardening.py similarity index 100% rename from tests/test_api_hardening.py rename to tests/unit/test_api_hardening.py diff --git a/tests/test_api_unit.py b/tests/unit/test_api_unit.py similarity index 99% rename from tests/test_api_unit.py rename to tests/unit/test_api_unit.py index b217142..ca33159 100644 --- a/tests/test_api_unit.py +++ b/tests/unit/test_api_unit.py @@ -154,7 +154,7 @@ def test_identifiers_are_never_string_formatted(self): import ast import pathlib - root = pathlib.Path(__file__).resolve().parents[1] / "api" + root = pathlib.Path(__file__).resolve().parents[2] / "api" offenders = [] for path in root.rglob("*.py"): tree = ast.parse(path.read_text(encoding="utf-8")) diff --git a/tests/test_ci_quality_gate.py b/tests/unit/test_ci_quality_gate.py similarity index 99% rename from tests/test_ci_quality_gate.py rename to tests/unit/test_ci_quality_gate.py index 2991b2e..7d9df96 100644 --- a/tests/test_ci_quality_gate.py +++ b/tests/unit/test_ci_quality_gate.py @@ -12,7 +12,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.regression] -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] WORKFLOW = ROOT / ".github" / "workflows" / "quality-gate.yml" GAP_ANALYSIS = ROOT / "docs/GAP_ANALYSIS.md" @@ -48,7 +48,6 @@ def test_triggers_on_push_to_main(self): self.assertIn("main", push_branches) - class TestAllJobsHaveConsistentStructure(unittest.TestCase): """All jobs must have checkout, python setup, and artifact upload.""" @@ -94,7 +93,6 @@ def test_all_jobs_install_dev_deps(self): ) - class TestGapAnalysisG2Accepted(unittest.TestCase): """G2 is accepted as out-of-scope, not closed. Checks the decision is recorded in docs/GAP_ANALYSIS.md. diff --git a/tests/test_csv_utilise.py b/tests/unit/test_csv_utilise.py similarity index 98% rename from tests/test_csv_utilise.py rename to tests/unit/test_csv_utilise.py index 834e23d..7aeee41 100644 --- a/tests/test_csv_utilise.py +++ b/tests/unit/test_csv_utilise.py @@ -15,7 +15,7 @@ pytestmark = pytest.mark.unit -SCRIPT = Path(__file__).resolve().parents[1] / "build" / "csv_utilise.sh" +SCRIPT = Path(__file__).resolve().parents[2] / "build" / "csv_utilise.sh" def _find_bash(): diff --git a/tests/test_csv_validator.py b/tests/unit/test_csv_validator.py similarity index 98% rename from tests/test_csv_validator.py rename to tests/unit/test_csv_validator.py index f8ad4a3..c26fee2 100644 --- a/tests/test_csv_validator.py +++ b/tests/unit/test_csv_validator.py @@ -18,7 +18,7 @@ class CsvValidatorTests(unittest.TestCase): def setUp(self): - self.validator = Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py" + self.validator = Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py" def run_validator(self, env): result = subprocess.run( diff --git a/tests/test_evals_runner.py b/tests/unit/test_evals_runner.py similarity index 98% rename from tests/test_evals_runner.py rename to tests/unit/test_evals_runner.py index 8ed9840..b37aa8c 100644 --- a/tests/test_evals_runner.py +++ b/tests/unit/test_evals_runner.py @@ -17,7 +17,7 @@ pytestmark = pytest.mark.unit -PROJECT_ROOT = Path(__file__).resolve().parents[1] +PROJECT_ROOT = Path(__file__).resolve().parents[2] RUNNER_PATH = PROJECT_ROOT / "evals" / "runner.py" spec = importlib.util.spec_from_file_location("evals_runner", RUNNER_PATH) diff --git a/tests/test_evals_tier_e_x.py b/tests/unit/test_evals_tier_e_x.py similarity index 97% rename from tests/test_evals_tier_e_x.py rename to tests/unit/test_evals_tier_e_x.py index 51efa99..f63d45a 100644 --- a/tests/test_evals_tier_e_x.py +++ b/tests/unit/test_evals_tier_e_x.py @@ -17,7 +17,7 @@ import pytest -sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "evals")) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "evals")) import runner # noqa: E402 @@ -168,7 +168,7 @@ def test_sqlite_internal_tables_are_excluded(self): def test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file(self): import json - root = Path(__file__).resolve().parent.parent + root = Path(__file__).resolve().parents[2] ddl = (root / "build" / "schema" / "sqlite" / "te_core_schema.sql").read_text(encoding="utf-8") exp = json.loads( @@ -185,7 +185,7 @@ def test_known_missing_tables_are_genuinely_absent(self): # these tables to the DDL but forgets to move it out of # known_missing_tables, the file would misreport the shortfall. import json - root = Path(__file__).resolve().parent.parent + root = Path(__file__).resolve().parents[2] ddl = (root / "build" / "schema" / "sqlite" / "te_core_schema.sql").read_text(encoding="utf-8") exp = json.loads( @@ -202,7 +202,7 @@ def test_known_missing_tables_are_genuinely_absent(self): def test_missing_plus_present_accounts_for_all_twelve_tables(self): import json - root = Path(__file__).resolve().parent.parent + root = Path(__file__).resolve().parents[2] exp = json.loads( (root / "evals" / "expected" / "tier_x" / "01_sqlite_schema_structure.json").read_text(encoding="utf-8") diff --git a/tests/test_preflight_ps1.py b/tests/unit/test_preflight_ps1.py similarity index 98% rename from tests/test_preflight_ps1.py rename to tests/unit/test_preflight_ps1.py index b6e3619..e35d1bd 100644 --- a/tests/test_preflight_ps1.py +++ b/tests/unit/test_preflight_ps1.py @@ -19,7 +19,7 @@ import pytest -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "preflight.ps1" WINDOWS_PS = shutil.which("powershell") if sys.platform == "win32" else None @@ -201,4 +201,4 @@ def test_port_label_follows_pgport(): def test_pg_checks_are_optional_never_fail(): r = _run([], extra_env={"PGHOST": "localhost", "PGPORT": "5999", "PGPASSWORD": "x"}) pg = r.stdout.split("PostgreSQL")[-1].split("Summary:")[0] - assert "FAIL" not in pg, r.stdout + r.stderr \ No newline at end of file + assert "FAIL" not in pg, r.stdout + r.stderr diff --git a/tests/test_regression.py b/tests/unit/test_regression.py similarity index 99% rename from tests/test_regression.py rename to tests/unit/test_regression.py index 536772f..814a0c1 100644 --- a/tests/test_regression.py +++ b/tests/unit/test_regression.py @@ -15,7 +15,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.regression] -VALIDATOR = Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py" +VALIDATOR = Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py" def _run(env: dict) -> subprocess.CompletedProcess: diff --git a/tests/test_security.py b/tests/unit/test_security.py similarity index 99% rename from tests/test_security.py rename to tests/unit/test_security.py index cbd8cb6..286c2bd 100644 --- a/tests/test_security.py +++ b/tests/unit/test_security.py @@ -14,7 +14,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.security] -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] # ── Python source files to scan ─────────────────────────────────────────────── PY_SOURCE = [ diff --git a/tests/test_service_loader.py b/tests/unit/test_service_loader.py similarity index 100% rename from tests/test_service_loader.py rename to tests/unit/test_service_loader.py diff --git a/tests/test_snapshot.py b/tests/unit/test_snapshot.py similarity index 92% rename from tests/test_snapshot.py rename to tests/unit/test_snapshot.py index 805b551..332b0d8 100644 --- a/tests/test_snapshot.py +++ b/tests/unit/test_snapshot.py @@ -4,8 +4,8 @@ file. On subsequent runs it compares the actual output against the stored file and fails if they differ. - UPDATE_SNAPSHOTS=1 pytest -m snapshot tests/test_snapshot.py # regenerate - pytest -m snapshot tests/test_snapshot.py # compare + UPDATE_SNAPSHOTS=1 pytest -m snapshot tests/unit/test_snapshot.py # regenerate + pytest -m snapshot tests/unit/test_snapshot.py # compare """ import os import subprocess @@ -18,7 +18,7 @@ pytestmark = [pytest.mark.unit, pytest.mark.snapshot] -VALIDATOR = Path(__file__).resolve().parents[1] / "build" / "csv" / "validator.py" +VALIDATOR = Path(__file__).resolve().parents[2] / "build" / "csv" / "validator.py" SNAPSHOTS_DIR = Path(__file__).resolve().parent / "snapshots" UPDATE = os.environ.get("UPDATE_SNAPSHOTS", "").lower() in ("1", "true", "yes") @@ -95,7 +95,7 @@ def test_all_valid_no_skips(self): self.assertEqual(exit_code, 0) _assert_snapshot(valid_text, "all_valid_expected_valid.csv") # skip file should have only a header (or be empty) - skip_lines = [l for l in skip_text.splitlines() if l.strip()] + skip_lines = [ln for ln in skip_text.splitlines() if ln.strip()] self.assertLessEqual(len(skip_lines), 1, "Expected at most a header row in skip.csv when all rows are valid")