diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 821ae9a..4bfab3c 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -41,6 +41,10 @@ jobs: - name: Component health check run: make health + # #169: test catalogue freshness — fails if docs/TEST_CASES.md is stale. + - name: Test catalogue check + run: python3 scripts/test_catalogue.py --check + # Tier X (cross-engine) is deliberately absent: multi-engine support is # deferred by ADR-0001 (PostgreSQL first), so this pipeline is PostgreSQL # only. The scenario still exists and passes — run it with `make @@ -165,6 +169,18 @@ jobs: PGDATABASE: te_mgmt_dev run: python3 scripts/test_report.py --strict + # #194: measure api/services coverage with per-loader floors. + - name: api/services coverage gate + env: + PGDATABASE: te_mgmt_dev + run: | + python -m pytest tests/ \ + --cov=api/services \ + --cov-report=json:coverage-services.json \ + --cov-report=term-missing \ + -q --no-header 2>&1 | tail -20 + python3 scripts/check_coverage_floors.py coverage-services.json + - name: Upload eval reports if: always() uses: actions/upload-artifact@v7 diff --git a/README.md b/README.md index 74bf629..8cf1d6a 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ PostgreDataMigrationAppWithCSVLoader/ │ ├── env.dev.example ← dev environment template ├── docs/ENV_VARIABLES.md ← all environment variables documented -├── docs/API_INTEGRATION.md ← frontend ↔ API endpoint mapping +├── docs/API_INTEGRATION.md ← API design notes: auth, known gaps, migrations ├── docs/QUICKSTART.md ← 10-minute setup guide └── docs/VCRM.md ← business requirement traceability ``` diff --git a/api/ratelimit.py b/api/ratelimit.py index d1c4876..2bd8255 100644 --- a/api/ratelimit.py +++ b/api/ratelimit.py @@ -88,17 +88,16 @@ def reset(self) -> None: def _client_id(request: Request) -> str: - """Identify the caller. + """Identify the caller by a trusted attribute. - Prefers the API key (a deployed caller's stable identity) over the socket - address, because several callers can share one NAT address. Falls back to - the peer IP. ``X-Forwarded-For`` is deliberately ignored: it is - client-controlled, so trusting it would let anyone bypass the limit by - varying the header. + Uses the peer IP as the rate-limit identity. The API key is NOT used here + because it is checked before authentication: an attacker could rotate + arbitrary X-API-Key values to get a fresh rate-limit bucket each time, + bypassing brute-force protection (#183). + + ``X-Forwarded-For`` is deliberately ignored: it is client-controlled, so + trusting it would let anyone bypass the limit by varying the header. """ - api_key = request.headers.get("x-api-key") - if api_key: - return f"key:{api_key[:8]}" client = request.client return f"ip:{client.host}" if client and client.host else "ip:unknown" diff --git a/api/routers/csv_routes.py b/api/routers/csv_routes.py index 898abb3..f4d5192 100644 --- a/api/routers/csv_routes.py +++ b/api/routers/csv_routes.py @@ -2,6 +2,8 @@ from __future__ import annotations +import base64 +import binascii import logging from urllib.parse import quote @@ -35,6 +37,11 @@ class PreviewRequest(BaseModel): content: str = Field(min_length=1) +class HeaderRename(BaseModel): + original: str = Field(max_length=255) + renamed: str = Field(default="", max_length=128) # blank keeps the original + + class UploadRequest(BaseModel): fileName: str = Field(min_length=1, max_length=255) content: str = Field(min_length=1) @@ -42,6 +49,10 @@ class UploadRequest(BaseModel): overwrite: bool = False mode: str = "dynamic" # "dynamic" | "te" targetTable: str | None = None # required when mode == "te" + headerMapping: list[HeaderRename] | None = Field(default=None, max_length=1000) # dynamic mode only (#186) + # The file's bytes as the browser read them, base64. Kept as the preserved + # original so its hash matches the uploader's copy; `content` is still what loads (#185). + contentBase64: str | None = None @router.post("/preview", dependencies=[Depends(allow("preview"))]) @@ -69,21 +80,40 @@ def preview(req: PreviewRequest) -> dict: return result +def _decode_original(b64: str | None) -> bytes | None: + if b64 is None: + return None + # base64 inflates by 4/3; refuse before decoding rather than after. + if len(b64) > settings.MAX_UPLOAD_BYTES * 4 // 3 + 4: + raise HTTPException(413, "File too large") + try: + return base64.b64decode(b64, validate=True) + except (binascii.Error, ValueError): + raise HTTPException(422, "contentBase64 is not valid base64") from None + + @router.post("/upload", dependencies=[Depends(allow("upload"))]) -def upload(req: UploadRequest) -> dict: +def upload(req: UploadRequest, session: dict | None = Depends(tenant_scope)) -> dict: if len(req.content.encode("utf-8")) > settings.MAX_UPLOAD_BYTES: raise HTTPException(413, "File too large") + original = _decode_original(req.contentBase64) if req.mode == "te": if not req.targetTable: raise HTTPException(422, "targetTable is required when mode is 'te'") - result = upload_te(req.fileName, req.content, req.targetTable) + if req.headerMapping: + # T&E columns must match the fixed schema exactly; a rename could + # only be ignored, which is the silent failure #186 describes. + raise HTTPException(422, "headerMapping is only supported in 'dynamic' mode") + result = upload_te(req.fileName, req.content, req.targetTable, session=session) elif req.mode == "dynamic": - result = upload_dynamic(req.fileName, req.content, req.types, req.overwrite) + mapping = [m.model_dump() for m in req.headerMapping] if req.headerMapping else None + result = upload_dynamic(req.fileName, req.content, req.types, req.overwrite, + session=session, header_mapping=mapping) else: raise HTTPException(422, f"Unknown mode '{req.mode}' — use 'dynamic' or 'te'") # M3b (#107): keep the original and its outcome, whatever the outcome was. - upload_id = record_upload(req.fileName, req.content, req.mode, result) + upload_id = record_upload(req.fileName, req.content, req.mode, result, original=original) result["uploadId"] = None if upload_id is None else str(upload_id) # O-04: record the load so audit_log covers creations, not just deletions. diff --git a/api/services/csv_parse.py b/api/services/csv_parse.py index 04f9eff..7cda87c 100644 --- a/api/services/csv_parse.py +++ b/api/services/csv_parse.py @@ -133,21 +133,7 @@ def sanitize_columns(headers: list[str]) -> list[str]: counter: dict[str, int] = {} out: list[str] = [] for idx, h in enumerate(headers): - base = (h or f"column_{idx + 1}").lower().strip() - base = re.sub(r"[^a-z0-9_]+", "_", base) - # Check reserved names (all of which have a leading underscore, e.g. - # "_id") before stripping underscores — otherwise "_id" strips to - # "id" first and never matches _RESERVED, silently losing the - # collision guard against the real system columns. - if base in _RESERVED: - base = f"{base}_col" - base = base.strip("_") - if not base: - base = f"column_{idx + 1}" - if base[0].isdigit(): - base = f"col_{base}" - if len(base) > 55: - base = base[:55] + base = _base_column_name(h, idx) name = base while name in assigned: counter[base] = counter.get(base, 0) + 1 @@ -157,6 +143,62 @@ def sanitize_columns(headers: list[str]) -> list[str]: return out +def _base_column_name(header: str, idx: int) -> str: + """One header's safe identifier, before de-duplication against its siblings.""" + base = (header or f"column_{idx + 1}").lower().strip() + base = re.sub(r"[^a-z0-9_]+", "_", base) + # Check reserved names (all of which have a leading underscore, e.g. + # "_id") before stripping underscores — otherwise "_id" strips to + # "id" first and never matches _RESERVED, silently losing the + # collision guard against the real system columns. + if base in _RESERVED: + base = f"{base}_col" + base = base.strip("_") + if not base: + base = f"column_{idx + 1}" + if base[0].isdigit(): + base = f"col_{base}" + if len(base) > 55: + base = base[:55] + return base + + +def apply_header_mapping( + headers: list[str], mapping: list[dict] | None +) -> tuple[list[str], str | None]: + """Apply user-confirmed header renames (#186) to the raw header row. + + ``mapping`` is a list of ``{"original": ..., "renamed": ...}``. A blank + ``renamed`` keeps the original. Returns ``(headers, error)``; on error the + headers are returned unchanged and nothing should be loaded. A rename is + refused, rather than silently bent into ``name_2``, when it would collide + with another column — the user asked for that exact name. + """ + if not mapping: + return headers, None + + renames: dict[str, str] = {} + for entry in mapping: + original = entry["original"] + if original in renames: + return headers, f'Header mapping lists "{original}" more than once.' + if original not in headers: + return headers, f'Header mapping refers to "{original}", which is not a header in this file.' + renames[original] = (entry.get("renamed") or "").strip() + + mapped = [renames.get(h) or h for h in headers] + renamed_idx = {i for i, h in enumerate(headers) if renames.get(h)} + by_base: dict[str, list[int]] = {} + for i, h in enumerate(mapped): + by_base.setdefault(_base_column_name(h, i), []).append(i) + for base, idxs in by_base.items(): + if len(idxs) > 1 and renamed_idx.intersection(idxs): + clash = next(i for i in idxs if i in renamed_idx) + return headers, (f'Renaming a column to "{mapped[clash]}" would give two columns the ' + f'name "{base}". Choose a different name.') + return mapped, None + + def extra_cells(raw: list[str], width: int) -> list[str]: """Non-empty cells beyond the header width (#82). diff --git a/api/services/destructive_ops.py b/api/services/destructive_ops.py new file mode 100644 index 0000000..0725a35 --- /dev/null +++ b/api/services/destructive_ops.py @@ -0,0 +1,57 @@ +"""Centralised authorisation for destructive database operations (#175, #170). + +Every code path that drops a user-data table must go through one of these +functions. This prevents the auth bypass where upload-with-overwrite or +T&E mode-switch silently dropped tables without checking delete permission. +""" + +from __future__ import annotations + +from fastapi import HTTPException +from psycopg2 import sql + +from api.auth import PERMISSIONS + + +def authorized_drop_table(cur, session: dict | None, schema: str, table_name: str) -> None: + """Drop a table only if the caller holds delete_file permission. + + ``session`` is ``None`` for API-key (non-session) requests, which are + allowed through (they have full access until per-key roles exist). + Raises HTTP 403 if the session's role lacks ``delete_file``. + """ + _require_delete(session) + cur.execute( + sql.SQL("DROP TABLE IF EXISTS {}.{}").format( + sql.Identifier(schema), sql.Identifier(table_name) + ) + ) + + +def safe_cleanup_prior(cur, session: dict | None, schema: str, + table_name: str, mode: str) -> None: + """Drop a prior dynamic-mode table during overwrite, with auth check. + + Only drops when the prior entry is dynamic mode and the table name starts + with ``csv_`` (T&E-mode entries point at shared tables that must not be + dropped here). + """ + _require_delete(session) + if mode == "dynamic" and table_name.startswith("csv_"): + cur.execute( + sql.SQL("DROP TABLE IF EXISTS {}.{}").format( + sql.Identifier(schema), sql.Identifier(table_name) + ) + ) + + +def _require_delete(session: dict | None) -> None: + """Raise 403 if the session's role cannot delete files.""" + if session is None: + return + roles = PERMISSIONS["delete_file"] + if session["role"] not in roles: + raise HTTPException( + status_code=403, + detail=f"Role '{session['role']}' may not delete or overwrite files", + ) diff --git a/api/services/dynamic_loader.py b/api/services/dynamic_loader.py index 4cb49ec..b8464d2 100644 --- a/api/services/dynamic_loader.py +++ b/api/services/dynamic_loader.py @@ -10,6 +10,7 @@ from __future__ import annotations import hashlib +import json import logging import time @@ -20,8 +21,10 @@ from api.config import settings from api.tenancy import uploads_schema from api.db import Conn +from api.services.destructive_ops import safe_cleanup_prior from api.services.csv_parse import ( ALLOWED_TYPES, + apply_header_mapping, cast_value, extra_cells, parse_csv, @@ -54,6 +57,9 @@ def upload_dynamic( content: str, types: list[str] | None, overwrite: bool, + *, + session: dict | None = None, + header_mapping: list[dict] | None = None, ) -> dict: logs: list[dict] = [] _log(logs, "receive", f'Received "{file_name}" ({len(content)} chars)') @@ -85,6 +91,17 @@ def upload_dynamic( "logs": logs, } + # #186: honour confirmed header renames; the column name in the table is + # the renamed one, not the CSV's original. + mapped_headers, mapping_error = apply_header_mapping(rows[0], header_mapping) + if mapping_error: + _log(logs, "header_mapping", mapping_error, "error") + return {"status": "error", "reason": "invalid_header_mapping", "message": mapping_error, "logs": logs} + if header_mapping: + renamed_count = sum(1 for a, b in zip(rows[0], mapped_headers) if a != b) + _log(logs, "header_mapping", f"Renamed {renamed_count} column header(s)", count=renamed_count) + rows = [mapped_headers] + rows[1:] + # BUG-027 guard: reject files above the configured row cap before doing any # per-row work. `rows` includes the header, so subtract one. data_row_count = len(rows) - 1 @@ -107,7 +124,7 @@ def upload_dynamic( schema = uploads_schema() try: - return _do_upload(file_name, types, overwrite, logs, schema, rows, file_hash) + return _do_upload(file_name, types, overwrite, logs, schema, rows, file_hash, session=session) except psycopg2.errors.UniqueViolation: # #81: an identical concurrent upload won a race that _do_upload could # not settle itself (e.g. both creating csv_ at once). Nothing @@ -142,6 +159,8 @@ def _do_upload( schema: str, rows: list[list[str]], file_hash: str, + *, + session: dict | None = None, ) -> dict: replaced_file_name: str | None = None with Conn() as conn: @@ -168,16 +187,7 @@ def _do_upload( "logs": logs, } _log(logs, "overwrite", f'Overwriting previous upload "{name_match[1]}"', "warn") - # table_name is only a bare identifier owned by this schema for - # dynamic-mode rows (csv_); TE-mode rows store "schema.table" - # pointing at a shared T&E table that must never be dropped here. - # Same guard as te_loader._cleanup_prior_registry_entries. - if name_match[5] == "dynamic" and name_match[2].startswith("csv_"): - cur.execute( - sql.SQL("DROP TABLE IF EXISTS {}.{}").format( - sql.Identifier(schema), sql.Identifier(name_match[2]) - ) - ) + safe_cleanup_prior(cur, session, schema, name_match[2], name_match[5]) cur.execute( sql.SQL("DELETE FROM {}.csv_files WHERE id = %s").format(sql.Identifier(schema)), (name_match[0],), @@ -206,13 +216,7 @@ def _do_upload( "logs": logs, } _log(logs, "overwrite", f'Overwriting previous content match "{content_match[1]}"', "warn") - # Same TE-vs-dynamic guard as above. - if content_match[4] == "dynamic" and content_match[2].startswith("csv_"): - cur.execute( - sql.SQL("DROP TABLE IF EXISTS {}.{}").format( - sql.Identifier(schema), sql.Identifier(content_match[2]) - ) - ) + safe_cleanup_prior(cur, session, schema, content_match[2], content_match[4]) cur.execute( sql.SQL("DELETE FROM {}.csv_files WHERE id = %s").format(sql.Identifier(schema)), (content_match[0],), @@ -301,10 +305,11 @@ def _do_upload( values.append(val) if failed: continue - # BUG-022: ASCII unit separator (U+001F) between cells prevents - # ["ab","cd"] and ["a","bcd"] from hashing to the same value. - row_hash = hashlib.sha256( - "\x1f".join(raw_joined).encode("utf-8") + # #178: JSON serialization prevents ambiguous concatenation. + # U+001F join was vulnerable: cells containing U+001F caused + # distinct rows to hash identically (silent data loss). + row_hash = "v2:" + hashlib.sha256( + json.dumps(raw_joined, ensure_ascii=False, separators=(",", ":")).encode("utf-8") ).hexdigest() if row_hash in seen: duplicates += 1 diff --git a/api/services/originals.py b/api/services/originals.py index 94ce358..f0d6860 100644 --- a/api/services/originals.py +++ b/api/services/originals.py @@ -36,9 +36,15 @@ def outcome(result: dict) -> str: return _STATUS.get(status, "failed") -def record_upload(file_name: str, content: str, mode: str, result: dict) -> int | None: - """Store the original and its outcome. Returns the upload id, or None if it couldn't be stored.""" - data = content.encode("utf-8") +def record_upload(file_name: str, content: str, mode: str, result: dict, *, + original: bytes | None = None) -> int | None: + """Store the original and its outcome. Returns the upload id, or None if it couldn't be stored. + + ``original`` is the file's bytes exactly as uploaded (#185); it is what gets hashed and + kept. Without it the decoded ``content`` re-encoded as UTF-8 stands in, which loses a + BOM or any invalid bytes. ``content`` still feeds the parsed ``source_rows`` view. + """ + data = original if original is not None else content.encode("utf-8") ok = result.get("status") == "ok" try: with Conn() as conn: diff --git a/api/services/review.py b/api/services/review.py index cb2a5a7..f8f8e25 100644 --- a/api/services/review.py +++ b/api/services/review.py @@ -31,18 +31,24 @@ def _s(): def _upload(cur, upload_id: int) -> dict: cur.execute(sql.SQL( - "SELECT u.file_name, u.columns, u.column_types, h.cells, " + "SELECT u.file_name, u.columns, u.column_types, h.cells, u.mode, u.row_errors, " " (SELECT count(*) FROM {s}.source_rows r WHERE r.upload_id = u.id AND r.row_number > 0) " "FROM {s}.uploads u LEFT JOIN {s}.source_rows h ON h.upload_id = u.id AND h.row_number = 0 " "WHERE u.id = %s").format(s=_s()), (upload_id,)) row = cur.fetchone() if row is None: raise ReviewError(404, "No such upload") - name, columns, types, header, total = row + name, columns, types, header, mode, stored_errors, total = row columns = columns or sanitize_columns(header or []) - # Uploads stored before 0008, or in T&E mode, have no types: every value counts as text. + # Uploads stored before 0008, or T&E uploads before #190, have no types: every value counts as text. types = types if types and len(types) == len(columns) else ["text"] * len(columns) - return {"fileName": name, "columns": columns, "types": types, "total": total} + # #190: T&E rows the database rejected (CHECK, FK, UNIQUE, ...). Errors with a column are + # type failures, which _validate finds again; these carry only the database's reason. + # ponytail: stored list is capped at MAX_ROW_ERRORS_REPORTED, rows past the cap show no + # database reason; edited rows aren't re-checked against constraints until M6's import. + db_errors = {e["rowNumber"]: e["reason"] for e in (stored_errors or []) + if mode == "te" and "column" not in e and "rowNumber" in e} + return {"fileName": name, "columns": columns, "types": types, "total": total, "_dbErrors": db_errors} def _validate(cells: list[str], raw: list[str], columns: list[str], types: list[str]) -> list[dict]: @@ -92,12 +98,15 @@ def _rows(cur, upload_id: int, meta: dict, numbers: list[int] | None = None, original = (list(raw) + [""] * width)[:width] current = [edits.get(n, {}).get(c, v) for c, v in enumerate(original)] decision = decisions.get(n) + errors = _validate(current, raw, meta["columns"], meta["types"]) + if n in meta["_dbErrors"] and n not in edits: + errors.append({"column": "(database)", "value": "", "reason": meta["_dbErrors"][n]}) out.append({ "rowNumber": n, "original": original, "current": current, "editedColumns": sorted(edits.get(n, {})), "decision": decision["decision"] if decision else "include", "decisionReason": decision["reason"] if decision else None, - "errors": _validate(current, raw, meta["columns"], meta["types"]), + "errors": errors, }) return out @@ -111,7 +120,8 @@ def page(cur, upload_id: int, offset: int, limit: int) -> dict: " (SELECT count(DISTINCT row_number) FROM {s}.cell_edits WHERE upload_id = %s)").format(s=_s()), (upload_id, upload_id)) excluded, edited = cur.fetchone() - return {**meta, "offset": offset, "limit": limit, "excludedRows": excluded, "editedRows": edited, + public = {k: v for k, v in meta.items() if k != "_dbErrors"} + return {**public, "offset": offset, "limit": limit, "excludedRows": excluded, "editedRows": edited, "rows": _rows(cur, upload_id, meta, offset=offset, limit=limit)} diff --git a/api/services/te_loader.py b/api/services/te_loader.py index be213ce..20b8681 100644 --- a/api/services/te_loader.py +++ b/api/services/te_loader.py @@ -29,6 +29,7 @@ from api.config import TE_TABLES, settings from api.tenancy import te_schema, uploads_schema from api.db import Conn +from api.services.destructive_ops import _require_delete from api.services.csv_parse import ( cast_value, extra_cells, @@ -112,7 +113,7 @@ def _cleanup_prior_registry_entries(cur, file_name, file_hash): ) -def upload_te(file_name: str, content: str, target_table: str) -> dict: +def upload_te(file_name: str, content: str, target_table: str, *, session: dict | None = None) -> dict: logs: list[dict] = [] _log(logs, "receive", f'Received "{file_name}" for T&E table "{target_table}"') @@ -214,9 +215,23 @@ def _err(row_number: int, column: str, value: str, reason: str) -> None: "info", ) + # Detect reupload before inserting so the all-failed rollback + # does not undo the registry cleanup (#170 + reupload compat). + file_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + cur.execute( + sql.SQL( + "SELECT 1 FROM {s}.csv_files WHERE file_name = %s OR file_hash = %s" + ).format(s=sql.Identifier(uploads_schema())), + (file_name, file_hash), + ) + is_reupload = cur.fetchone() is not None + if is_reupload: + _require_delete(session) + _cleanup_prior_registry_entries(cur, file_name, file_hash) + # Pre-validate every row client-side; drop failures into row_errors. failed_row_count = 0 - to_insert: list[list] = [] + to_insert: list[tuple[int, list]] = [] # (original_csv_row, values) for r, raw in enumerate(data_rows): row_number = r + 1 values: list = [] @@ -232,8 +247,6 @@ def _err(row_number: int, column: str, value: str, reason: str) -> None: values.append(None) continue if col_type is None: - # Let PG handle it; pass the raw string through and - # rely on the SAVEPOINT fallback below to catch errors. values.append(cell) continue ok, val, reason = cast_value(cell, col_type) @@ -244,7 +257,7 @@ def _err(row_number: int, column: str, value: str, reason: str) -> None: break values.append(val) if not failed: - to_insert.append(values) + to_insert.append((row_number, values)) _log( logs, @@ -257,25 +270,20 @@ def _err(row_number: int, column: str, value: str, reason: str) -> None: # Batch insert survivors. If a constraint (FK, UNIQUE, CHECK on an # unmapped column, etc.) trips, roll back the chunk and fall back - # to per-row to identify the bad row(s) — same shape as before but - # only runs on the ~few rows that had DB-only failure modes. + # to per-row to identify the bad row(s). chunk_size = 500 for start in range(0, len(to_insert), chunk_size): chunk = to_insert[start : start + chunk_size] + chunk_values = [v for _, v in chunk] cur.execute("SAVEPOINT chunk_sp") try: - execute_values(cur, batch_insert_stmt.as_string(cur), chunk) + execute_values(cur, batch_insert_stmt.as_string(cur), chunk_values) cur.execute("RELEASE SAVEPOINT chunk_sp") inserted += len(chunk) except Exception: # noqa: BLE001 — DB-level constraint: retry per-row cur.execute("ROLLBACK TO SAVEPOINT chunk_sp") cur.execute("RELEASE SAVEPOINT chunk_sp") - for i, values in enumerate(chunk): - # We don't know which original CSV row this maps to - # because to_insert dropped failures — reconstruct via - # position in the surviving list. Row numbers reported - # here are the position within the successful subset; - # good enough to point the user at the record. + for orig_row_num, values in chunk: cur.execute("SAVEPOINT row_sp") try: cur.execute(insert_stmt, values) @@ -285,20 +293,30 @@ def _err(row_number: int, column: str, value: str, reason: str) -> None: failed_row_count += 1 if len(row_errors) < settings.MAX_ROW_ERRORS_REPORTED: row_errors.append({ - "rowNumber": start + i + 1, + "rowNumber": orig_row_num, "reason": str(exc).split("\n")[0], }) finally: cur.execute("RELEASE SAVEPOINT row_sp") - # Register the load in the shared registry (mode='te') - file_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() - # T&E loads may be re-run; clear any prior registry entry for this - # file name or identical content before re-registering. Uses - # _cleanup_prior_registry_entries (not a plain DELETE) so a prior - # *dynamic*-mode upload of the same file also has its csv_ - # table dropped instead of orphaned. - _cleanup_prior_registry_entries(cur, file_name, file_hash) + # #170: if every row failed, nothing was migrated. Roll back and + # preserve any prior dataset rather than cleaning it up. + if data_rows and inserted == 0 and not is_reupload: + conn.rollback() + msg = f"None of the {len(data_rows)} rows could be imported into {target_table}, so nothing was saved." + if row_errors: + e = row_errors[0] + msg += f' First problem: row {e["rowNumber"]}, {e.get("column", "")}: {e.get("reason", "")}' + _log(logs, "error", msg, "error") + return {"status": "error", "message": msg, "rowErrors": row_errors, + "totalRows": len(data_rows), "failedRows": failed_row_count, + "insertedRows": 0, "columns": columns, "logs": logs} + + # Register the load in the shared registry (mode='te'). + # Reupload cleanup was done before inserts; for first uploads + # clean any stale entries now (usually a no-op). + if not is_reupload: + _cleanup_prior_registry_entries(cur, file_name, file_hash) try: cur.execute( sql.SQL( @@ -332,7 +350,9 @@ def _err(row_number: int, column: str, value: str, reason: str) -> None: "duplicateRowsSkipped": 0, "failedRows": failed_row_count, "columns": columns, - "types": [], + # #190: stored with the upload so review revalidates against the target's types. + # Columns Postgres casts itself (uuid, enums, ...) review as text. + "types": [t or "text" for t in col_types], "rowErrors": row_errors, "logs": logs, } diff --git a/build/setup.sh b/build/setup.sh index d58494b..15302c3 100644 --- a/build/setup.sh +++ b/build/setup.sh @@ -340,7 +340,12 @@ esac # ══════════════════════════════════════════════════════════════════════════════ header "Step 5 — Writing Configuration" -cat > "$CONFIG_LOCAL" << EOF +# #171: Atomic write — write to a temp file first, then mv. If the heredoc +# expansion fails (unset variable under set -u), the prior config survives. +_config_tmp="${CONFIG_LOCAL}.new" +trap 'rm -f "$_config_tmp"' EXIT + +cat > "$_config_tmp" << EOF # ============================================================================= # config.local.env — Generated by setup.sh on $(date) # Engine: ${DB_ENGINE} @@ -473,6 +478,13 @@ SEED_STAGING=${SEED_STAGING} SEED_PROD=${SEED_PROD} EOF +if [[ ! -s "$_config_tmp" ]]; then + error "Generated config is empty; prior config.local.env preserved." + rm -f "$_config_tmp" + exit 1 +fi +mv -f "$_config_tmp" "$CONFIG_LOCAL" + success "Configuration written to: config.local.env" # ── Summary ──────────────────────────────────────────────────────────────────── diff --git a/csv-table-hub-main/src/__tests__/export.test.ts b/csv-table-hub-main/src/__tests__/export.test.ts index 941c9a3..2f93ad5 100644 --- a/csv-table-hub-main/src/__tests__/export.test.ts +++ b/csv-table-hub-main/src/__tests__/export.test.ts @@ -234,6 +234,43 @@ describe("downloadErrorReport XLSX", () => { }); }); +// ─── downloadErrorReport — XLSX formula guard (#193) ───────────────────────── + +describe("downloadErrorReport XLSX formula guard", () => { + let cap: ReturnType; + + beforeEach(() => { cap = captureBlob(); }); + afterEach(() => { cap.reset(); }); + + it.each([ + ['=HYPERLINK("http://x","y")'], + ["+cmd|' /C calc'!A0"], + ["-2+3"], + ["@SUM(A1:A2)"], + ["\tcmd"], + ])("neutralises formula payload %s in XLSX", async (value) => { + const rows = buildErrorRows("data.csv", [{ rowNumber: 1, column: "c", value, reason: "test" }]); + downloadErrorReport("data.csv", rows, "xlsx"); + const buf = await cap.blob!.arrayBuffer(); + const wb = XLSX.read(buf, { type: "array" }); + const ws = wb.Sheets["Errors"]; + const parsed = XLSX.utils.sheet_to_json>(ws); + expect((parsed[0] as any).value).toBe(`'${value}`); + }); + + it("leaves plain negative numbers unchanged in XLSX", async () => { + const rows = buildErrorRows("data.csv", [ + { rowNumber: 1, column: "n", value: "-5", reason: "+3.2" }, + ]); + downloadErrorReport("data.csv", rows, "xlsx"); + const buf = await cap.blob!.arrayBuffer(); + const wb = XLSX.read(buf, { type: "array" }); + const parsed = XLSX.utils.sheet_to_json>(wb.Sheets["Errors"]); + expect((parsed[0] as any).value).toBe("-5"); + expect((parsed[0] as any).reason).toBe("+3.2"); + }); +}); + // ─── downloadAuditReport ────────────────────────────────────────────────────── describe("downloadAuditReport", () => { diff --git a/csv-table-hub-main/src/__tests__/file-bytes.test.ts b/csv-table-hub-main/src/__tests__/file-bytes.test.ts new file mode 100644 index 0000000..414aedd --- /dev/null +++ b/csv-table-hub-main/src/__tests__/file-bytes.test.ts @@ -0,0 +1,38 @@ +// #185 (BR-24): the preserved original is the file's bytes, not decoded text. + +import { describe, it, expect } from "vitest"; +import { bytesToBase64, readFileBase64 } from "@/lib/file-bytes"; + +function fromBase64(b64: string): Uint8Array { + return Uint8Array.from(atob(b64), (c) => c.charCodeAt(0)); +} + +describe("bytesToBase64", () => { + it("encodes arbitrary bytes, including ones that are not valid UTF-8", () => { + const bytes = new Uint8Array([0x61, 0x80, 0xff, 0x00, 0x62]); + expect(Array.from(fromBase64(bytesToBase64(bytes)))).toEqual(Array.from(bytes)); + }); + + it("handles input larger than one conversion chunk", () => { + const bytes = new Uint8Array(0x8000 * 3 + 17).map((_, i) => i % 256); + expect(fromBase64(bytesToBase64(bytes))).toEqual(bytes); + }); + + it("encodes an empty array as an empty string", () => { + expect(bytesToBase64(new Uint8Array())).toBe(""); + }); +}); + +describe("readFileBase64", () => { + it("keeps a UTF-8 BOM that readAsText would drop", async () => { + const bytes = new Uint8Array([0xef, 0xbb, 0xbf, ...new TextEncoder().encode("a,b\n1,2\n")]); + const out = fromBase64(await readFileBase64(new File([bytes], "bom.csv"))); + expect(Array.from(out.subarray(0, 3))).toEqual([0xef, 0xbb, 0xbf]); + expect(out).toEqual(bytes); + }); + + it("keeps a lone 0x80 byte that readAsText would replace with U+FFFD", async () => { + const bytes = new Uint8Array([...new TextEncoder().encode("a\n1"), 0x80, 0x0a]); + expect(fromBase64(await readFileBase64(new File([bytes], "bad.csv")))).toEqual(bytes); + }); +}); diff --git a/csv-table-hub-main/src/__tests__/review-table.test.tsx b/csv-table-hub-main/src/__tests__/review-table.test.tsx index c208e0e..19a03f5 100644 --- a/csv-table-hub-main/src/__tests__/review-table.test.tsx +++ b/csv-table-hub-main/src/__tests__/review-table.test.tsx @@ -32,7 +32,11 @@ const page = { row(2, ["x", "b"], { errors: [{ column: "n", value: "x", reason: '"x" is not a whole number' }], }), - row(3, ["3", "c"], { decision: "exclude", decisionReason: "duplicate of row 1" }), + row(3, ["3", "c"], { + decision: "exclude", + decisionReason: "duplicate of row 1", + errors: [{ column: "(database)", value: "", reason: "violates check constraint" }], + }), ], }; @@ -82,6 +86,8 @@ it("shows every row with its validation, edits and decision", async () => { ); expect(screen.getByText("Invalid")).toBeInTheDocument(); expect(screen.getByText("Excluded")).toHaveAttribute("title", "duplicate of row 1"); + // #190: a reason not tied to a column (the database rejected the row) is shown beside it. + expect(screen.getByText("violates check constraint")).toBeInTheDocument(); }); it("a reviewer excludes a row with a required reason", async () => { diff --git a/csv-table-hub-main/src/__tests__/uploader-state-machine.test.ts b/csv-table-hub-main/src/__tests__/uploader-state-machine.test.ts index 1bb8ba0..a1fc01c 100644 --- a/csv-table-hub-main/src/__tests__/uploader-state-machine.test.ts +++ b/csv-table-hub-main/src/__tests__/uploader-state-machine.test.ts @@ -189,4 +189,27 @@ describe("Uploader state machine", () => { const call = uploadCsv.mock.calls[0][0]; expect(call.overwrite).toBe(true); }); + + it("sends the file's raw bytes alongside the decoded text (#185)", async () => { + uploadCsv.mockResolvedValue({ + status: "ok", + fileId: "1", + tableName: "csv_abc123", + totalRows: 1, + insertedRows: 1, + duplicateRowsSkipped: 0, + failedRows: 0, + columns: ["id", "name"], + types: ["int8", "text"], + rowErrors: [], + logs: [], + }); + const raw = new Uint8Array([0xef, 0xbb, 0xbf, ...new TextEncoder().encode("id,name\n1,foo\n")]); + await dropAndConfirm(new File([raw], "bom.csv", { type: "text/csv" })); + + await waitFor(() => expect(uploadCsv).toHaveBeenCalled()); + const call = uploadCsv.mock.calls[0][0]; + expect(Array.from(Uint8Array.from(atob(call.contentBase64), (c) => c.charCodeAt(0)))).toEqual(Array.from(raw)); + expect(call.content.startsWith("id,name")).toBe(true); // the BOM is not in the decoded text + }); }); diff --git a/csv-table-hub-main/src/components/review-table.tsx b/csv-table-hub-main/src/components/review-table.tsx index 2e21b15..8924aa4 100644 --- a/csv-table-hub-main/src/components/review-table.tsx +++ b/csv-table-hub-main/src/components/review-table.tsx @@ -161,7 +161,8 @@ function ReviewLine({ }) { const excluded = row.decision === "exclude"; const errorFor = (name: string) => row.errors.find((e) => e.column === name); - const extra = errorFor("(beyond header)"); + // Reasons not tied to a column: "(beyond header)" cells, or "(database)" for a row the target rejected (#190). + const extra = row.errors.find((e) => !columns.includes(e.column)); return ( diff --git a/csv-table-hub-main/src/lib/csv.functions.ts b/csv-table-hub-main/src/lib/csv.functions.ts index 79db888..2afd19a 100644 --- a/csv-table-hub-main/src/lib/csv.functions.ts +++ b/csv-table-hub-main/src/lib/csv.functions.ts @@ -156,6 +156,7 @@ export async function uploadCsv(input: { mode?: "dynamic" | "te"; targetTable?: string; headerMapping?: { original: string; renamed: string }[]; + contentBase64?: string; }): Promise { return apiFetch("/api/csv/upload", uploadResultSchema, { method: "POST", @@ -167,6 +168,7 @@ export async function uploadCsv(input: { mode: input.mode ?? "dynamic", targetTable: input.targetTable, headerMapping: input.headerMapping, + contentBase64: input.contentBase64, }), }); } diff --git a/csv-table-hub-main/src/lib/export.ts b/csv-table-hub-main/src/lib/export.ts index 4ae4806..e5a5e90 100644 --- a/csv-table-hub-main/src/lib/export.ts +++ b/csv-table-hub-main/src/lib/export.ts @@ -62,12 +62,29 @@ function toCsvBlob(headers: string[], rows: Record[]): Blob { // ─── XLSX ───────────────────────────────────────────────────────────────────── +function sanitizeForXlsx( + rows: Record[], + headers: string[], +): Record[] { + return rows.map((r) => { + const out: Record = {}; + for (const h of headers) { + let v = r[h]; + if (typeof v === "string" && FORMULA_START.test(v) && !PLAIN_NUMBER.test(v)) { + v = `'${v}`; + } + out[h] = v; + } + return out; + }); +} + function toXlsxBlob( sheetName: string, headers: string[], rows: Record[], ): Blob { - const ws = XLSX.utils.json_to_sheet(rows, { header: headers }); + const ws = XLSX.utils.json_to_sheet(sanitizeForXlsx(rows, headers), { header: headers }); const wb = XLSX.utils.book_new(); XLSX.utils.book_append_sheet(wb, ws, sheetName.slice(0, 31)); const buf = XLSX.write(wb, { type: "array", bookType: "xlsx" }); diff --git a/csv-table-hub-main/src/lib/file-bytes.ts b/csv-table-hub-main/src/lib/file-bytes.ts new file mode 100644 index 0000000..fd3ef07 --- /dev/null +++ b/csv-table-hub-main/src/lib/file-bytes.ts @@ -0,0 +1,23 @@ +// The file's bytes exactly as on disk, base64-encoded (#185; BR-24). +// FileReader.readAsText drops a BOM and replaces invalid bytes, so the text the +// loaders receive can't be hashed or stored as "the original". The API keeps +// these bytes as the preserved original and still loads from the decoded text. + +const CHUNK = 0x8000; // keeps String.fromCharCode's argument list well under engine limits + +export function bytesToBase64(bytes: Uint8Array): string { + let binary = ""; + for (let i = 0; i < bytes.length; i += CHUNK) { + binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK)); + } + return btoa(binary); +} + +export function readFileBase64(file: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => reject(reader.error ?? new Error("Could not read the file.")); + reader.onload = () => resolve(bytesToBase64(new Uint8Array(reader.result as ArrayBuffer))); + reader.readAsArrayBuffer(file); + }); +} diff --git a/csv-table-hub-main/src/routes/_authenticated/index.tsx b/csv-table-hub-main/src/routes/_authenticated/index.tsx index d492d06..e03807d 100644 --- a/csv-table-hub-main/src/routes/_authenticated/index.tsx +++ b/csv-table-hub-main/src/routes/_authenticated/index.tsx @@ -34,6 +34,7 @@ import { } from "@/lib/mapping-templates"; import { downloadErrorReport, buildErrorRows, type ExportFormat } from "@/lib/export"; import type { Job, JobStatus } from "@/lib/job-types"; +import { readFileBase64 } from "@/lib/file-bytes"; import { formatLocalTime, formatUtc } from "@/lib/utils"; import { classifyUploadResult, classifyThrownError, buildCancelPatch, buildCancelToastMessage, @@ -169,6 +170,8 @@ export function Uploader() { reader.readAsText(file); }); if (ctrl.signal.aborted) return; + const contentBase64 = await readFileBase64(file); // #185: keep the original bytes, not the decoded text + if (ctrl.signal.aborted) return; updateJob(job.id,{status:"uploading",progress:75}); await new Promise(r=>setTimeout(r,30)); if (ctrl.signal.aborted) return; @@ -181,7 +184,7 @@ export function Uploader() { const headerMapping = isTe ? undefined : mapping; const res = await uploadCsv({ - fileName:job.name, content, types, overwrite, + fileName:job.name, content, contentBase64, types, overwrite, headerMapping: headerMapping?.map(c=>({original:c.original,renamed:c.renamed})), mode: job.mode, targetTable: job.targetTable, }); diff --git a/docs/API_INTEGRATION.md b/docs/API_INTEGRATION.md index eccf0a5..28dac12 100644 --- a/docs/API_INTEGRATION.md +++ b/docs/API_INTEGRATION.md @@ -5,7 +5,7 @@ Notes on the FastAPI backend in `api/`, which connects the React frontend in monorepo) to PostgreSQL. > **See also** the **Web UI + REST API** section of `README.md` for the -> user-facing quickstart (two-terminal launch, endpoint table, env vars). This +> user-facing quickstart (two-terminal launch, env vars). This > file is the deeper design/integration doc — for how the backend is put > together and what's still open. @@ -34,14 +34,11 @@ T&E table, which drives the mode picker in the UI. ## Endpoints -| Method | Path | Purpose | -|---|---|---| -| GET | `/api/health` | Liveness; reports `ok` or `degraded`, never errors | -| POST | `/api/csv/preview` | Parse and type-detect without writing | -| POST | `/api/csv/upload` | Load (`mode: dynamic \| te`) | -| GET | `/api/csv/files` | Registered uploads — drives "Migrated files" | -| GET | `/api/csv/tables/{table}/rows` | Row preview, `limit` 1–200 | -| DELETE | `/api/csv/files/{id}` | Remove registration; drops the table in dynamic mode | +This file deliberately does not list routes: a hand-kept table fell behind the +code (#207). The route list is generated from the code, so it cannot drift. +With the API running, open `http://localhost:8000/docs` (Swagger UI) or fetch +`/openapi.json`. Neither needs a key. Routers live in `api/routers/` +(`auth`, `csv`, `review`, `te`, `audit`). CSV content is sent as a JSON string, not multipart. diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 39ee1e1..a882851 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -296,7 +296,7 @@ PostgreDataMigrationAppWithCSVLoader/ │ ├── env.dev.example ← dev environment template ├── docs/ENV_VARIABLES.md ← all env vars documented -├── docs/API_INTEGRATION.md ← frontend ↔ API endpoint mapping +├── docs/API_INTEGRATION.md ← API design notes: auth, known gaps, migrations └── docs/VCRM.md ← business requirement traceability ``` @@ -307,7 +307,8 @@ PostgreDataMigrationAppWithCSVLoader/ - `docs/STARTER_GUIDE.md` — orientation, key concepts, and a guided first walkthrough (dynamic mode vs T&E mode, running the test suite end to end) - `docs/ENV_VARIABLES.md` — full list of every environment variable -- `docs/API_INTEGRATION.md` — frontend ↔ backend endpoint mapping and gaps +- `docs/API_INTEGRATION.md` — API design notes: auth model, known gaps, migrations + (route list: `http://localhost:8000/docs` with the API running) - `docs/VCRM.md` — business requirement traceability (142 SQL assertions) - `docs/ARCHITECTURE.md` — three-layer architecture rationale - `evals/USAGE.md` — eval runner flags and CI integration diff --git a/docs/STARTER_GUIDE.md b/docs/STARTER_GUIDE.md index f91ce8e..fa44d89 100644 --- a/docs/STARTER_GUIDE.md +++ b/docs/STARTER_GUIDE.md @@ -49,8 +49,8 @@ which mode was used: real schema" path. Implemented in `api/services/te_loader.py`. Both modes go through the same `POST /api/csv/upload` endpoint, with a -`mode: "dynamic" | "te"` field distinguishing them — see -`docs/API_INTEGRATION.md` for the full request/response shapes. +`mode: "dynamic" | "te"` field distinguishing them — see the Swagger UI at +`http://localhost:8000/docs` (API running) for the full request/response shapes. ## Multi-environment by design @@ -83,26 +83,33 @@ Once the app is running locally, this sequence exercises both modes and the test suite in about ten minutes — a good way to confirm your setup actually works, not just that servers started. -1. **Upload a CSV in dynamic mode.** Any small CSV works — three columns, +1. **Sign in.** The app has required authentication since ART Milestone 2, so + `http://localhost:8080` opens on the sign-in page, not the upload area. + Follow QUICKSTART's **"Create your sign-in (first run only)"** section to + create a tenant admin, then sign in with it. +2. **Upload a CSV in dynamic mode.** Any small CSV works — three columns, a few rows. Drop it on the upload area, preview it, confirm the import. Check the Migrated Files table for the new `csv_` table and row count. -2. **Upload a CSV in T&E mode.** Pick one of the 12 T&E tables (e.g. +3. **Upload a CSV in T&E mode.** Pick one of the 12 T&E tables (e.g. `organisations`), match your CSV's headers to its columns, and import. This exercises the validation path dynamic mode skips. -3. **Re-upload the same file.** Confirm duplicate detection kicks in - (content-hash based, not filename based). -4. **Run the fast test suite** to confirm the backend is sane: +4. **Re-upload the dynamic-mode file from step 2.** Confirm duplicate + detection kicks in — the upload is rejected with "Duplicate file" on either + a matching filename or a matching content hash. Dynamic mode only: a T&E + re-upload replaces the previous registry entry instead of refusing, since + per-tenant and per-record dedup is still M5 scope (#109). +5. **Run the fast test suite** to confirm the backend is sane: ```powershell python -m pytest tests\ -m "unit or regression or security or snapshot" -q ``` -5. **Run the frontend tests:** +6. **Run the frontend tests:** ```powershell cd csv-table-hub-main npm test ``` -If all five pass/succeed, your environment is a faithful match for CI. +If all six pass/succeed, your environment is a faithful match for CI. --- @@ -112,7 +119,8 @@ If all five pass/succeed, your environment is a faithful match for CI. |---|---| | Set up my dev environment from scratch | `docs/QUICKSTART.md` | | Understand the repo's folder structure and why it's split that way | `docs/ARCHITECTURE.md` | -| See every frontend↔backend API call and its shape | `docs/API_INTEGRATION.md` | +| See every API route and its request/response shape | `http://localhost:8000/docs` while the API is running (Swagger UI) | +| Understand how the API is put together, its auth model, known gaps and migrations | `docs/API_INTEGRATION.md` | | Look up an environment variable | `docs/ENV_VARIABLES.md` | | Understand Dev/Test/Staging/Prod differences | `docs/DATABASE_ENVIRONMENTS.md` | | Deploy to Azure | `docs/AZURE_DEPLOY.md` (non-prod) / `docs/PROD_DEPLOY.md` (prod) | diff --git a/docs/TEST_CASES.md b/docs/TEST_CASES.md new file mode 100644 index 0000000..500fc4d --- /dev/null +++ b/docs/TEST_CASES.md @@ -0,0 +1,1607 @@ +# Test Catalogue + +> Auto-generated by `scripts/test_catalogue.py` — do not edit by hand. +> 726 tests across 37 files. + +To regenerate: `python3 scripts/test_catalogue.py` + +## `tests/test_accounts.py` + +| Test | Markers | Objective | +|---|---|---| +| `Passwords.test_round_trip` | unit | — | + +| `Passwords.test_wrong_password_fails` | unit | — | + +| `Passwords.test_same_password_hashes_differently` | unit | — | + +| `Passwords.test_malformed_hash_fails_closed` | unit | — | + +| `SignIn.test_login_returns_a_session_for_the_tenant_and_role` | integration | — | + +| `SignIn.test_every_login_failure_is_the_same_401` | integration | — | + +| `SignIn.test_inactive_user_cannot_sign_in_and_loses_live_sessions` | integration | — | + +| `SignIn.test_logout_revokes_the_session` | integration | — | + +| `SignIn.test_expired_session_is_rejected` | integration | — | + +| `SignIn.test_removing_the_membership_ends_the_session` | integration | — | + +| `SignIn.test_missing_or_malformed_token_is_401` | integration | — | + +| `SignIn.test_tokens_are_stored_only_as_hashes` | integration | — | + +| `Lockout.test_eleventh_attempt_is_refused_even_with_the_right_password` | integration | — | + +| `Lockout.test_one_short_of_the_limit_still_signs_in_and_resets_the_count` | integration | — | + +| `Lockout.test_sign_in_works_again_after_the_lock_expires` | integration | — | + +| `Lockout.test_failures_outside_the_window_start_a_new_count` | integration | — | + +| `test_round_trip` | — | — | + +| `test_wrong_password_fails` | — | — | + +| `test_same_password_hashes_differently` | — | — | + +| `test_malformed_hash_fails_closed` | — | — | + +| `test_login_returns_a_session_for_the_tenant_and_role` | — | — | + +| `test_every_login_failure_is_the_same_401` | — | — | + +| `test_inactive_user_cannot_sign_in_and_loses_live_sessions` | — | — | + +| `test_logout_revokes_the_session` | — | — | + +| `test_expired_session_is_rejected` | — | — | + +| `test_removing_the_membership_ends_the_session` | — | — | + +| `test_missing_or_malformed_token_is_401` | — | — | + +| `test_tokens_are_stored_only_as_hashes` | — | — | + +| `test_eleventh_attempt_is_refused_even_with_the_right_password` | — | — | + +| `test_one_short_of_the_limit_still_signs_in_and_resets_the_count` | — | — | + +| `test_sign_in_works_again_after_the_lock_expires` | — | — | + +| `test_failures_outside_the_window_start_a_new_count` | — | — | + +## `tests/test_api_coverage.py` + +| Test | Markers | Objective | +|---|---|---| +| `TeTablesConfig.test_te_tables_has_twelve_entries` | unit | — | + +| `TeTablesConfig.test_te_tables_are_all_strings` | unit | — | + +| `TeTablesConfig.test_te_tables_has_no_duplicates` | unit | — | + +| `CorsHeaders.test_preflight_returns_cors_headers` | unit | — | + +| `CorsHeaders.test_preflight_allows_put_for_review_changes` | unit | #108: the browser's review screen sends PUT; without it the preflight fails. | + +| `CorsHeaders.test_cors_allows_configured_origin` | unit | — | + +| `CorsHeaders.test_default_origins_include_vite_dev_port` | unit | Issue #43: the Vite dev server runs on :8080 and must be allowed by default. | + +| `CorsHeaders.test_cors_rejects_unconfigured_origin` | unit | — | + +| `TeTablesEndpoint.test_returns_200` | integration | — | + +| `TeTablesEndpoint.test_returns_list_of_twelve` | integration | — | + +| `TeTablesEndpoint.test_each_entry_has_required_keys` | integration | — | + +| `TeTablesEndpoint.test_table_names_match_whitelist` | integration | — | + +| `TeTablesEndpoint.test_all_tables_exist_after_deploy` | integration | — | + +| `TeTablesEndpoint.test_row_count_is_non_negative_integer` | integration | — | + +| `PreviewTeTableMatch.test_preview_returns_te_match_for_matching_columns` | integration | — | + +| `PreviewTeTableMatch.test_preview_returns_null_for_non_matching_columns` | integration | — | + +| `PreviewTeTableMatch.test_te_table_match_key_present_in_ok_response` | integration | — | + +| `DeleteEndpoint.test_delete_returns_correct_shape` | integration | — | + +| `DeleteEndpoint.test_delete_nonexistent_id_returns_404` | integration | — | + +| `DeleteEndpoint.test_delete_dynamic_drops_backing_table` | integration | — | + +| `DeleteEndpoint.test_delete_te_only_removes_registry` | integration | — | + +| `DeleteEndpoint.test_deleted_file_no_longer_listed` | integration | — | + +| `AuditLogEndpoint.test_delete_appears_in_audit_log` | integration | — | + +| `OverwriteUpload.test_overwrite_replaces_existing_file` | integration | — | + +| `OverwriteUpload.test_overwrite_false_refuses_duplicate` | integration | — | + +| `OverwriteUpload.test_overwrite_creates_new_table` | integration | — | + +| `OverwriteUpload.test_overwrite_response_has_fields` | integration | — | + +| `TeUploadPipeline.test_successful_upload_to_organisations` | integration | — | + +| `TeUploadPipeline.test_response_shape_matches_contract` | integration | — | + +| `TeUploadPipeline.test_column_mismatch_returns_error` | integration | — | + +| `TeUploadPipeline.test_row_level_error_reported` | integration | — | + +| `TeUploadPipeline.test_te_upload_registers_in_csv_files` | integration | — | + +| `TeUploadPipeline.test_enum_constraint_violation_reported_per_row` | integration | An invalid enum value must produce a per-row error, not a 500. | + +| `TeUploadPipeline.test_null_in_not_null_column_reported_per_row` | integration | An empty required field sent as NULL must fail per-row, not crash. | + +| `TeUploadPipeline.test_unknown_target_table_returns_error` | integration | Uploading to a table not in TE_TABLES must return status=error. | + +| `TeUploadPipeline.test_te_reupload_same_file_replaces_registry` | integration | Re-uploading the same CSV in T&E mode replaces the registry entry. | + +| `TeUploadPipeline.test_te_client_side_type_validation_reports_column_and_value` | integration | BUG-028: a bad value on a client-validatable column (BOOLEAN here) | + +| `TeUploadPipeline.test_te_upload_rejects_over_row_cap` | integration | BUG-027 for T&E mode: a CSV with more than settings.MAX_ROWS data | + +| `DeleteGateUnit.test_delete_blocked_returns_403` | unit | — | + +| `test_te_tables_has_twelve_entries` | — | — | + +| `test_te_tables_are_all_strings` | — | — | + +| `test_te_tables_has_no_duplicates` | — | — | + +| `test_preflight_returns_cors_headers` | — | — | + +| `test_preflight_allows_put_for_review_changes` | — | #108: the browser's review screen sends PUT; without it the preflight fails. | + +| `test_cors_allows_configured_origin` | — | — | + +| `test_default_origins_include_vite_dev_port` | — | Issue #43: the Vite dev server runs on :8080 and must be allowed by default. | + +| `test_cors_rejects_unconfigured_origin` | — | — | + +| `test_returns_200` | — | — | + +| `test_returns_list_of_twelve` | — | — | + +| `test_each_entry_has_required_keys` | — | — | + +| `test_table_names_match_whitelist` | — | — | + +| `test_all_tables_exist_after_deploy` | — | — | + +| `test_row_count_is_non_negative_integer` | — | — | + +| `test_preview_returns_te_match_for_matching_columns` | — | — | + +| `test_preview_returns_null_for_non_matching_columns` | — | — | + +| `test_te_table_match_key_present_in_ok_response` | — | — | + +| `test_delete_returns_correct_shape` | — | — | + +| `test_delete_nonexistent_id_returns_404` | — | — | + +| `test_delete_dynamic_drops_backing_table` | — | — | + +| `test_delete_te_only_removes_registry` | — | — | + +| `test_deleted_file_no_longer_listed` | — | — | + +| `test_delete_appears_in_audit_log` | — | — | + +| `test_overwrite_replaces_existing_file` | — | — | + +| `test_overwrite_false_refuses_duplicate` | — | — | + +| `test_overwrite_creates_new_table` | — | — | + +| `test_overwrite_response_has_fields` | — | — | + +| `test_successful_upload_to_organisations` | — | — | + +| `test_response_shape_matches_contract` | — | — | + +| `test_column_mismatch_returns_error` | — | — | + +| `test_row_level_error_reported` | — | — | + +| `test_te_upload_registers_in_csv_files` | — | — | + +| `test_enum_constraint_violation_reported_per_row` | — | An invalid enum value must produce a per-row error, not a 500. | + +| `test_null_in_not_null_column_reported_per_row` | — | An empty required field sent as NULL must fail per-row, not crash. | + +| `test_unknown_target_table_returns_error` | — | Uploading to a table not in TE_TABLES must return status=error. | + +| `test_te_reupload_same_file_replaces_registry` | — | Re-uploading the same CSV in T&E mode replaces the registry entry. | + +| `test_te_client_side_type_validation_reports_column_and_value` | — | BUG-028: a bad value on a client-validatable column (BOOLEAN here) | + +| `test_te_upload_rejects_over_row_cap` | — | BUG-027 for T&E mode: a CSV with more than settings.MAX_ROWS data | + +| `test_delete_blocked_returns_403` | — | — | + +## `tests/test_api_hardening.py` + +| Test | Markers | Objective | +|---|---|---| +| `AuthFailsClosed.test_dev_may_run_without_a_key` | unit | — | + +| `AuthFailsClosed.test_any_key_is_accepted_in_dev` | unit | — | + +| `AuthFailsClosed.test_prod_without_a_key_refuses_to_start` | unit | — | + +| `AuthFailsClosed.test_staging_and_test_also_refuse` | unit | — | + +| `AuthFailsClosed.test_deployed_env_with_a_key_starts` | unit | — | + +| `RateLimiterAlgorithm.test_requests_under_budget_are_allowed` | unit | — | + +| `RateLimiterAlgorithm.test_request_over_budget_is_refused` | unit | — | + +| `RateLimiterAlgorithm.test_retry_after_points_at_the_oldest_hit_leaving_the_window` | unit | — | + +| `RateLimiterAlgorithm.test_window_expiry_frees_the_budget` | unit | — | + +| `RateLimiterAlgorithm.test_zero_budget_disables_the_limit` | unit | — | + +| `RateLimiterAlgorithm.test_read_and_write_scopes_are_independent` | unit | — | + +| `RateLimiterAlgorithm.test_clients_are_independent` | unit | — | + +| `RateLimiterAlgorithm.test_bucket_table_is_bounded` | unit | — | + +| `RateLimiterAlgorithm.test_reset_clears_state` | unit | — | + +| `RateLimitMiddleware.test_exceeding_the_write_budget_returns_429_with_retry_after` | unit | — | + +| `RateLimitMiddleware.test_health_is_never_rate_limited` | unit | — | + +| `RateLimitMiddleware.test_openapi_is_never_rate_limited` | unit | — | + +| `RateLimitMiddleware.test_disabled_budget_allows_everything` | unit | — | + +| `RateLimitMiddleware.test_forwarded_for_cannot_be_used_to_evade_the_limit` | unit | — | + +| `AuditWriterContract.test_only_known_actions_are_accepted` | unit | — | + +| `AuditWriterContract.test_unknown_action_raises_before_any_sql_runs` | unit | — | + +| `AuditWriterContract.test_valid_action_reaches_the_cursor_with_bound_parameters` | unit | — | + +| `AuditWriterContract.test_file_id_is_coerced_from_the_loaders_string_form` | unit | — | + +| `AuditWriterContract.test_unparseable_file_id_degrades_to_null_rather_than_raising` | unit | — | + +| `test_dev_may_run_without_a_key` | — | — | + +| `test_any_key_is_accepted_in_dev` | — | — | + +| `test_prod_without_a_key_refuses_to_start` | — | — | + +| `test_staging_and_test_also_refuse` | — | — | + +| `test_deployed_env_with_a_key_starts` | — | — | + +| `test_requests_under_budget_are_allowed` | — | — | + +| `test_request_over_budget_is_refused` | — | — | + +| `test_retry_after_points_at_the_oldest_hit_leaving_the_window` | — | — | + +| `test_window_expiry_frees_the_budget` | — | — | + +| `test_zero_budget_disables_the_limit` | — | — | + +| `test_read_and_write_scopes_are_independent` | — | — | + +| `test_clients_are_independent` | — | — | + +| `test_bucket_table_is_bounded` | — | — | + +| `test_reset_clears_state` | — | — | + +| `test_exceeding_the_write_budget_returns_429_with_retry_after` | — | — | + +| `test_health_is_never_rate_limited` | — | — | + +| `test_openapi_is_never_rate_limited` | — | — | + +| `test_disabled_budget_allows_everything` | — | — | + +| `test_forwarded_for_cannot_be_used_to_evade_the_limit` | — | — | + +| `test_only_known_actions_are_accepted` | — | — | + +| `test_unknown_action_raises_before_any_sql_runs` | — | — | + +| `test_valid_action_reaches_the_cursor_with_bound_parameters` | — | — | + +| `test_file_id_is_coerced_from_the_loaders_string_form` | — | — | + +| `test_unparseable_file_id_degrades_to_null_rather_than_raising` | — | — | + +## `tests/test_api_integration.py` + +| Test | Markers | Objective | +|---|---|---| +| `HealthContract.test_health_always_returns_200` | integration | — | + +| `HealthContract.test_health_status_is_ok_or_degraded` | integration | — | + +| `HealthContract.test_openapi_schema_is_served` | integration | — | + +| `HealthContract.test_health_is_not_behind_api_key` | integration | BUG-035 regression: /api/health must be reachable without X-API-Key | + +| `HealthContract.test_csv_endpoint_still_requires_api_key_when_set` | integration | BUG-035 counterpart: business endpoints must still be guarded. | + +| `CsvPipelineWithDatabase.test_preview_reports_columns` | integration | — | + +| `CsvPipelineWithDatabase.test_upload_then_list_then_read_rows` | integration | — | + +| `CsvPipelineWithDatabase.test_reupload_of_identical_content_is_refused` | integration | — | + +| `CsvPipelineWithDatabase.test_in_file_duplicate_rows_are_skipped` | integration | — | + +| `CsvPipelineWithDatabase.test_row_hash_does_not_collide_on_split_boundaries` | integration | BUG-022 regression: `["ab","cd"]` and `["a","bcd"]` used to hash to | + +| `CsvPipelineWithDatabase.test_row_hash_still_dedupes_identical_rows` | integration | Guard-rail for BUG-022 fix: the separator change must not break the | + +| `CsvPipelineWithDatabase.test_unregistered_table_is_404` | integration | — | + +| `CsvPipelineWithDatabase.test_injection_shaped_names_are_not_served` | integration | Names that pass the prefix guard must still 404 when unregistered. | + +| `CsvPipelineWithDatabase.test_core_te_tables_are_intact` | integration | A CSV upload must never disturb the 12 tables the SQL suite asserts. | + +| `CsvPipelineWithDatabase.test_audit_record_contains_required_fields` | integration | BR-15: GET /api/csv/files must return all required audit fields. | + +| `CsvPipelineWithDatabase.test_audit_record_row_count_matches_inserted_rows` | integration | BR-15: persisted row_count must match what the upload response reported. | + +| `CsvPipelineWithDatabase.test_audit_record_table_name_matches_upload_response` | integration | BR-15: persisted table_name must match what the upload response reported. | + +| `CsvPipelineWithDatabase.test_audit_record_has_iso_timestamp` | integration | BR-15: created_at must be a non-empty ISO timestamp. | + +| `CsvPipelineWithDatabase.test_failed_upload_leaves_no_audit_record` | integration | BR-15: a structurally invalid file must not create a registry entry. | + +| `AuditLogTrail.test_successful_upload_writes_an_audit_row` | integration | O-04: the gap this closes — a load used to leave no audit trace. | + +| `AuditLogTrail.test_audit_row_fields_match_the_upload_response` | integration | The row must identify what was loaded and where, not merely that | + +| `AuditLogTrail.test_audit_row_names_the_actor_and_the_chain_verifies` | integration | M3a (#107): who acted is recorded, and the hash chain checks out. | + +| `AuditLogTrail.test_upload_then_delete_leaves_both_rows` | integration | The full lifecycle. Before O-04 only the second half was recorded, | + +| `AuditLogTrail.test_duplicate_upload_writes_no_second_audit_row` | integration | Only a genuine load is audited. A refused duplicate changed nothing, | + +| `AuditLogTrail.test_failed_upload_writes_no_audit_row` | integration | A header-only file is rejected before any rows land, so there is | + +| `AuditLogTrail.test_audit_endpoint_reports_a_total_and_respects_limit` | integration | Pagination contract, so a grower table stays consultable. | + +| `PoolTimeout.test_exhausted_pool_raises_within_timeout` | integration | psycopg2's ThreadedConnectionPool.getconn() raises PoolError | + +| `test_health_always_returns_200` | — | — | + +| `test_health_status_is_ok_or_degraded` | — | — | + +| `test_openapi_schema_is_served` | — | — | + +| `test_health_is_not_behind_api_key` | — | BUG-035 regression: /api/health must be reachable without X-API-Key | + +| `test_csv_endpoint_still_requires_api_key_when_set` | — | BUG-035 counterpart: business endpoints must still be guarded. | + +| `test_preview_reports_columns` | — | — | + +| `test_upload_then_list_then_read_rows` | — | — | + +| `test_reupload_of_identical_content_is_refused` | — | — | + +| `test_in_file_duplicate_rows_are_skipped` | — | — | + +| `test_row_hash_does_not_collide_on_split_boundaries` | — | BUG-022 regression: `["ab","cd"]` and `["a","bcd"]` used to hash to | + +| `test_row_hash_still_dedupes_identical_rows` | — | Guard-rail for BUG-022 fix: the separator change must not break the | + +| `test_unregistered_table_is_404` | — | — | + +| `test_injection_shaped_names_are_not_served` | — | Names that pass the prefix guard must still 404 when unregistered. | + +| `test_core_te_tables_are_intact` | — | A CSV upload must never disturb the 12 tables the SQL suite asserts. | + +| `test_audit_record_contains_required_fields` | — | BR-15: GET /api/csv/files must return all required audit fields. | + +| `test_audit_record_row_count_matches_inserted_rows` | — | BR-15: persisted row_count must match what the upload response reported. | + +| `test_audit_record_table_name_matches_upload_response` | — | BR-15: persisted table_name must match what the upload response reported. | + +| `test_audit_record_has_iso_timestamp` | — | BR-15: created_at must be a non-empty ISO timestamp. | + +| `test_failed_upload_leaves_no_audit_record` | — | BR-15: a structurally invalid file must not create a registry entry. | + +| `test_successful_upload_writes_an_audit_row` | — | O-04: the gap this closes — a load used to leave no audit trace. | + +| `test_audit_row_fields_match_the_upload_response` | — | The row must identify what was loaded and where, not merely that | + +| `test_audit_row_names_the_actor_and_the_chain_verifies` | — | M3a (#107): who acted is recorded, and the hash chain checks out. | + +| `test_upload_then_delete_leaves_both_rows` | — | The full lifecycle. Before O-04 only the second half was recorded, | + +| `test_duplicate_upload_writes_no_second_audit_row` | — | Only a genuine load is audited. A refused duplicate changed nothing, | + +| `test_failed_upload_writes_no_audit_row` | — | A header-only file is rejected before any rows land, so there is | + +| `test_audit_endpoint_reports_a_total_and_respects_limit` | — | Pagination contract, so a grower table stays consultable. | + +| `test_exhausted_pool_raises_within_timeout` | — | psycopg2's ThreadedConnectionPool.getconn() raises PoolError | + +## `tests/test_api_unit.py` + +| Test | Markers | Objective | +|---|---|---| +| `UnitCsvParse.test_sanitize_columns_no_collision_on_stem_conflict` | unit | BUG-036 regression: ["foo","foo","foo_2"] must produce three | + +| `UnitCsvParse.test_sanitize_columns_dedupes_true_duplicates` | unit | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | + +| `UnitCsvParse.test_sanitize_columns_reserved_stems_get_suffix` | unit | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | + +| `RequestValidation.test_preview_rejects_empty_filename` | unit | — | + +| `RequestValidation.test_preview_rejects_empty_content` | unit | — | + +| `RequestValidation.test_preview_rejects_overlong_filename` | unit | — | + +| `RequestValidation.test_upload_rejects_unknown_mode` | unit | — | + +| `RequestValidation.test_upload_te_mode_requires_target_table` | unit | — | + +| `RequestValidation.test_oversized_upload_is_rejected` | unit | — | + +| `TableNameGuards.test_non_upload_tables_are_rejected` | security, unit | — | + +| `TableNameGuards.test_overlong_names_are_rejected` | security, unit | — | + +| `TableNameGuards.test_identifiers_are_never_string_formatted` | security, unit | Every SQL identifier must go through psycopg2.sql.Identifier. | + +| `test_sanitize_columns_no_collision_on_stem_conflict` | — | BUG-036 regression: ["foo","foo","foo_2"] must produce three | + +| `test_sanitize_columns_dedupes_true_duplicates` | — | Guard-rail for the BUG-036 fix: real duplicate stems still dedupe. | + +| `test_sanitize_columns_reserved_stems_get_suffix` | — | Guard-rail: reserved names (_id, _row_hash, _created_at) must be | + +| `test_preview_rejects_empty_filename` | — | — | + +| `test_preview_rejects_empty_content` | — | — | + +| `test_preview_rejects_overlong_filename` | — | — | + +| `test_upload_rejects_unknown_mode` | — | — | + +| `test_upload_te_mode_requires_target_table` | — | — | + +| `test_oversized_upload_is_rejected` | — | — | + +| `test_non_upload_tables_are_rejected` | — | — | + +| `test_overlong_names_are_rejected` | — | — | + +| `test_identifiers_are_never_string_formatted` | — | Every SQL identifier must go through psycopg2.sql.Identifier. | + +## `tests/test_audit_append_only.py` + +| Test | Markers | Objective | +|---|---|---| +| `AppendOnlyAudit.test_rows_written_before_the_upgrade_are_chained` | integration | — | + +| `AppendOnlyAudit.test_update_delete_and_truncate_are_rejected` | integration | — | + +| `AppendOnlyAudit.test_records_actor_tenant_and_values` | integration | — | + +| `AppendOnlyAudit.test_a_changed_row_breaks_the_chain_at_that_row` | integration | — | + +| `AppendOnlyAudit.test_a_removed_row_breaks_the_chain` | integration | — | + +| `AppendOnlyAudit.test_concurrent_writers_keep_one_unbroken_chain` | integration | — | + +| `AppendOnlyAudit.test_downgrade_removes_the_triggers_and_columns` | integration | — | + +| `test_rows_written_before_the_upgrade_are_chained` | — | — | + +| `test_update_delete_and_truncate_are_rejected` | — | — | + +| `test_records_actor_tenant_and_values` | — | — | + +| `test_a_changed_row_breaks_the_chain_at_that_row` | — | — | + +| `test_a_removed_row_breaks_the_chain` | — | — | + +| `test_concurrent_writers_keep_one_unbroken_chain` | — | — | + +| `test_downgrade_removes_the_triggers_and_columns` | — | — | + +## `tests/test_ci_quality_gate.py` + +| Test | Markers | Objective | +|---|---|---| +| `TestQualityGateStructure.test_has_free_tier_job` | — | — | + +| `TestQualityGateStructure.test_has_integration_postgres_job` | — | — | + +| `TestQualityGateStructure.test_triggers_on_pull_request` | — | — | + +| `TestQualityGateStructure.test_triggers_on_push_to_main` | — | — | + +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_checkout` | — | — | + +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_python_setup` | — | — | + +| `TestAllJobsHaveConsistentStructure.test_all_jobs_have_artifact_upload` | — | — | + +| `TestAllJobsHaveConsistentStructure.test_all_jobs_install_dev_deps` | — | — | + +| `TestGapAnalysisG2Accepted.test_g2_row_exists` | — | — | + +| `TestGapAnalysisG2Accepted.test_g2_is_accepted_or_deferred` | — | — | + +| `TestGapAnalysisG2Accepted.test_g2_section_exists` | — | — | + +| `TestGapAnalysisG2Accepted.test_g2_is_not_marked_closed` | — | — | + +| `test_has_free_tier_job` | — | — | + +| `test_has_integration_postgres_job` | — | — | + +| `test_triggers_on_pull_request` | — | — | + +| `test_triggers_on_push_to_main` | — | — | + +| `test_all_jobs_have_checkout` | — | — | + +| `test_all_jobs_have_python_setup` | — | — | + +| `test_all_jobs_have_artifact_upload` | — | — | + +| `test_all_jobs_install_dev_deps` | — | — | + +| `test_g2_row_exists` | — | — | + +| `test_g2_is_accepted_or_deferred` | — | — | + +| `test_g2_section_exists` | — | — | + +| `test_g2_is_not_marked_closed` | — | — | + +## `tests/test_csv_loader_arbitrary_shapes.py` + +| Test | Markers | Objective | +|---|---|---| +| `CsvLoaderArbitraryShapes.test_shape_tiny` | — | — | + +| `CsvLoaderArbitraryShapes.test_shape_medium` | — | — | + +| `CsvLoaderArbitraryShapes.test_shape_skinny` | — | — | + +| `test_shape_tiny` | — | — | + +| `test_shape_medium` | — | — | + +| `test_shape_skinny` | — | — | + +## `tests/test_csv_utilise.py` + +| Test | Markers | Objective | +|---|---|---| +| `CsvUtiliseArgumentParsing.test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | + +| `CsvUtiliseArgumentParsing.test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | + +| `CsvUtiliseArgumentParsing.test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | + +| `CsvUtiliseArgumentParsing.test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | + +| `CsvUtiliseArgumentParsing.test_export_requires_two_positional_args` | — | `export` needs both and → exit 1 otherwise. | + +| `CsvUtiliseArgumentParsing.test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | + +| `CsvUtiliseArgumentParsing.test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | + +| `CsvUtiliseArgumentParsing.test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | + +| `CsvUtiliseArgumentParsing.test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | + +| `test_no_args_shows_usage_and_exits_nonzero` | — | No args → exit 1 with usage banner on stdout. | + +| `test_unknown_command_exits_nonzero` | — | Unknown subcommand → exit 1 with 'Unknown command' on stderr. | + +| `test_describe_requires_table_arg` | — | `describe` with no positional table → exit 1 with a helpful message. | + +| `test_peek_requires_table_arg` | — | `peek` with no positional table → exit 1 with a helpful message. | + +| `test_export_requires_two_positional_args` | — | `export` needs both
and → exit 1 otherwise. | + +| `test_drop_requires_table_arg` | — | `drop` with no positional table → exit 1. | + +| `test_help_flag_exits_zero` | — | `--help` short-circuits to a 0 exit with the usage banner. | + +| `test_invalid_table_name_rejected` | — | Identifier validation rejects names with spaces or punctuation. | + +| `test_unimplemented_engine_returns_clear_error` | — | Engines other than postgresql exit 2 with a 'not implemented' message. | + +## `tests/test_csv_validator.py` + +| Test | Markers | Objective | +|---|---|---| +| `CsvValidatorTests.test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | + +| `CsvValidatorTests.test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | + +| `CsvValidatorTests.test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | + +| `CsvValidatorTests.test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | + +| `test_fails_when_required_env_vars_missing` | — | When CSV_FILE/VALID_CSV/SKIP_FILE are absent, the validator should exit 1 with 'Missing required environment variables' on stderr. | + +| `test_fails_when_csv_file_missing` | — | When CSV_FILE points to a non-existent path, the validator should exit 1 with 'CSV file not found' on stderr. | + +| `test_splits_valid_and_skipped_rows` | — | Given a 4-row CSV (2 valid, 1 empty row, 1 column-mismatch), valid.csv gets the 2 good rows and skip.csv gets 2 annotated rows with a _skip_reason column. | + +| `test_returns_error_when_no_valid_rows` | — | When all data rows are invalid, the validator should exit 1 with 'No valid rows found' on stderr. | + +## `tests/test_e2e_pipeline.py` + +| Test | Markers | Objective | +|---|---|---| +| `E2EPipelineValidateOnly.test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | + +| `E2EPipelineValidateOnly.test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | + +| `E2EPipelineValidateOnly.test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | + +| `E2EPipelineValidateOnly.test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | + +| `E2EPipelineWithDatabase.test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | + +| `E2EPipelineWithDatabase.test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | + +| `test_happy_path_two_valid_rows` | — | Standard 2-column CSV with 2 valid rows produces correct outputs. | + +| `test_mixed_rows_splits_correctly` | — | Mixed CSV produces separate valid and skip outputs. | + +| `test_all_invalid_rows_exit_nonzero` | — | A CSV where every data row fails validation must exit 1. | + +| `test_output_directory_is_created_automatically` | — | Validator must create the output directory if it does not exist. | + +| `test_tier_p_scenarios_all_pass` | — | All Tier P eval scenarios must pass when the runner is executed end-to-end. | + +| `test_tier_i_idempotency` | — | Tier I (idempotency) scenarios must pass against the running PostgreSQL. | + +## `tests/test_evals_runner.py` + +| Test | Markers | Objective | +|---|---|---| +| `EvalsRunnerTests.test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | + +| `EvalsRunnerTests.test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | + +| `EvalsRunnerTests.test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | + +| `EvalsRunnerTests.test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | + +| `EvalsRunnerTests.test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | + +| `EvalsRunnerTests.test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | + +| `EvalsRunnerTests.test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | + +| `test_load_expected_returns_none_for_missing_scenario` | — | _load_expected with an unknown scenario name should return None without raising. | + +| `test_discover_scenarios_filters_only_requested_name` | — | discover_scenarios with a name argument should return only the directory whose name matches exactly. | + +| `test_tier_p_reports_unknown_runner_action_as_failure` | — | An expected JSON with an unrecognised runner_action should produce a failed result containing 'Unknown runner_action' in errors. | + +| `test_tier_p_generated_invalid_utf8_skips_row_cleanly` | — | The 23_invalid_utf8_bytes eval scenario: an invalid-UTF-8 data row is | + +| `test_tier_i_fails_when_postgresql_is_unavailable` | — | When _can_connect_pg returns False, run_tier_i_scenario must FAIL (not skip): an unavailable prerequisite is a failure, so a green run always means the scenario really executed. | + +| `test_count_dev_rows_records_none_for_failed_count_query` | — | When the count query subprocess exits non-zero, _count_dev_rows should record None for every entry in DEV_SEED_TABLES. | + +| `test_have_psql_uses_path_lookup` | — | When shutil.which returns None, _have_psql should return False. | + +## `tests/test_evals_tier_e_x.py` + +| Test | Markers | Objective | +|---|---|---| +| `TierEDiffSignatures.test_identical_schemas_report_no_differences` | unit | — | + +| `TierEDiffSignatures.test_namespace_is_not_part_of_the_comparison` | unit | — | + +| `TierEDiffSignatures.test_missing_table_is_caught` | unit | — | + +| `TierEDiffSignatures.test_extra_table_is_caught` | unit | — | + +| `TierEDiffSignatures.test_added_column_is_caught` | unit | — | + +| `TierEDiffSignatures.test_changed_type_is_caught` | unit | — | + +| `TierEDiffSignatures.test_changed_nullability_is_caught` | unit | — | + +| `TierEDiffSignatures.test_reordered_columns_are_caught_as_drift` | unit | — | + +| `TierEDiffSignatures.test_env_targets_cover_all_four_environments` | unit | — | + +| `TierEColumnSignatureQuery.test_query_is_sent_on_stdin_not_via_dash_c` | unit | — | + +| `TierEColumnSignatureQuery.test_query_failure_is_reported_as_a_query_error` | unit | — | + +| `TierEColumnSignatureQuery.test_genuinely_empty_schema_still_fails` | unit | — | + +| `TierXSqliteStructure.test_placeholders_are_substituted` | unit | — | + +| `TierXSqliteStructure.test_every_known_table_has_a_substitution` | unit | — | + +| `TierXSqliteStructure.test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | unit | — | + +| `TierXSqliteStructure.test_structure_is_read_back_from_executed_ddl` | unit | — | + +| `TierXSqliteStructure.test_invalid_ddl_is_reported_rather_than_raised` | unit | — | + +| `TierXSqliteStructure.test_sqlite_internal_tables_are_excluded` | unit | — | + +| `TierXSqliteStructure.test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | unit | — | + +| `TierXSqliteStructure.test_known_missing_tables_are_genuinely_absent` | unit | — | + +| `TierXSqliteStructure.test_missing_plus_present_accounts_for_all_twelve_tables` | unit | — | + +| `TierRegistration.test_all_five_tiers_are_registered` | unit | — | + +| `test_identical_schemas_report_no_differences` | — | — | + +| `test_namespace_is_not_part_of_the_comparison` | — | — | + +| `test_missing_table_is_caught` | — | — | + +| `test_extra_table_is_caught` | — | — | + +| `test_added_column_is_caught` | — | — | + +| `test_changed_type_is_caught` | — | — | + +| `test_changed_nullability_is_caught` | — | — | + +| `test_reordered_columns_are_caught_as_drift` | — | — | + +| `test_env_targets_cover_all_four_environments` | — | — | + +| `test_query_is_sent_on_stdin_not_via_dash_c` | — | — | + +| `test_query_failure_is_reported_as_a_query_error` | — | — | + +| `test_genuinely_empty_schema_still_fails` | — | — | + +| `test_placeholders_are_substituted` | — | — | + +| `test_every_known_table_has_a_substitution` | — | — | + +| `test_unknown_placeholder_survives_so_the_scenario_can_flag_it` | — | — | + +| `test_structure_is_read_back_from_executed_ddl` | — | — | + +| `test_invalid_ddl_is_reported_rather_than_raised` | — | — | + +| `test_sqlite_internal_tables_are_excluded` | — | — | + +| `test_the_real_sqlite_ddl_executes_and_matches_the_expectation_file` | — | — | + +| `test_known_missing_tables_are_genuinely_absent` | — | — | + +| `test_missing_plus_present_accounts_for_all_twelve_tables` | — | — | + +| `test_all_five_tiers_are_registered` | — | — | + +## `tests/test_failed_uploads.py` + +| Test | Markers | Objective | +|---|---|---| +| `EveryRowFailed.test_every_row_failing_is_an_error_and_saves_nothing` | integration | — | + +| `EveryRowFailed.test_failed_overwrite_keeps_the_previous_upload` | integration | — | + +| `EveryRowFailed.test_partly_failed_upload_still_imports_and_reports_the_failures` | integration | — | + +| `test_every_row_failing_is_an_error_and_saves_nothing` | — | — | + +| `test_failed_overwrite_keeps_the_previous_upload` | — | — | + +| `test_partly_failed_upload_still_imports_and_reports_the_failures` | — | — | + +## `tests/test_header_mapping.py` + +| Test | Markers | Objective | +|---|---|---| +| `ApplyHeaderMapping.test_no_mapping_returns_headers_unchanged` | unit | — | + +| `ApplyHeaderMapping.test_rename_replaces_only_the_named_header` | unit | — | + +| `ApplyHeaderMapping.test_blank_rename_keeps_the_original` | unit | — | + +| `ApplyHeaderMapping.test_unknown_original_is_rejected` | unit | — | + +| `ApplyHeaderMapping.test_original_listed_twice_is_rejected` | unit | — | + +| `ApplyHeaderMapping.test_rename_colliding_with_an_untouched_column_is_rejected` | unit | — | + +| `ApplyHeaderMapping.test_collision_is_judged_after_sanitising` | unit | — | + +| `ApplyHeaderMapping.test_swapping_two_names_is_allowed` | unit | — | + +| `HeaderMappingOverTheApi.test_renamed_header_becomes_the_database_column` | integration | — | + +| `HeaderMappingOverTheApi.test_upload_without_a_mapping_keeps_csv_headers` | integration | — | + +| `HeaderMappingOverTheApi.test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | integration | — | + +| `HeaderMappingOverTheApi.test_rename_that_collides_with_another_column_is_an_error` | integration | — | + +| `HeaderMappingOverTheApi.test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | integration | — | + +| `test_no_mapping_returns_headers_unchanged` | — | — | + +| `test_rename_replaces_only_the_named_header` | — | — | + +| `test_blank_rename_keeps_the_original` | — | — | + +| `test_unknown_original_is_rejected` | — | — | + +| `test_original_listed_twice_is_rejected` | — | — | + +| `test_rename_colliding_with_an_untouched_column_is_rejected` | — | — | + +| `test_collision_is_judged_after_sanitising` | — | — | + +| `test_swapping_two_names_is_allowed` | — | — | + +| `test_renamed_header_becomes_the_database_column` | — | — | + +| `test_upload_without_a_mapping_keeps_csv_headers` | — | — | + +| `test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing` | — | — | + +| `test_rename_that_collides_with_another_column_is_an_error` | — | — | + +| `test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it` | — | — | + +## `tests/test_import_counts.py` + +| Test | Markers | Objective | +|---|---|---| +| `ImportCountsArePersisted.test_dynamic_import_persists_all_four_counts` | integration | — | + +| `ImportCountsArePersisted.test_te_import_persists_counts` | integration | — | + +| `test_dynamic_import_persists_all_four_counts` | — | — | + +| `test_te_import_persists_counts` | — | — | + +## `tests/test_issue_04_multi_file_upload.py` + +| Test | Markers | Objective | +|---|---|---| +| `MultipleFileUpload.test_many_files_upload_in_one_session` | integration | Uploading several files in sequence must all succeed. | + +| `MultipleFileUpload.test_every_uploaded_file_is_listed` | integration | All uploads appear in /api/csv/files — this drives 'Migrated files'. | + +| `MultipleFileUpload.test_each_file_is_independently_readable` | integration | Rows from one upload must not leak into another's table. | + +| `MultipleFileUpload.test_narrow_and_wide_files_both_load` | integration | One column and many columns must both be accepted. | + +| `MultipleFileUpload.test_files_with_different_schemas_coexist` | integration | Unrelated column sets must not collide — no shared fixed schema. | + +| `MultipleFileUpload.test_a_larger_file_loads_completely` | integration | Row count is not silently truncated. | + +| `test_many_files_upload_in_one_session` | — | Uploading several files in sequence must all succeed. | + +| `test_every_uploaded_file_is_listed` | — | All uploads appear in /api/csv/files — this drives 'Migrated files'. | + +| `test_each_file_is_independently_readable` | — | Rows from one upload must not leak into another's table. | + +| `test_narrow_and_wide_files_both_load` | — | One column and many columns must both be accepted. | + +| `test_files_with_different_schemas_coexist` | — | Unrelated column sets must not collide — no shared fixed schema. | + +| `test_a_larger_file_loads_completely` | — | Row count is not silently truncated. | + +## `tests/test_issue_05_import_summary.py` + +| Test | Markers | Objective | +|---|---|---| +| `ImportSummaryCounts.test_all_four_counts_are_returned` | integration | — | + +| `ImportSummaryCounts.test_total_rows_counts_data_rows_not_the_header` | integration | — | + +| `ImportSummaryCounts.test_created_count_matches_rows_actually_inserted` | integration | — | + +| `ImportSummaryCounts.test_duplicate_count_reflects_repeated_rows_in_the_file` | integration | Two identical rows plus one distinct: 1 duplicate skipped. | + +| `ImportSummaryCounts.test_failed_count_reflects_rows_that_could_not_be_cast` | integration | With an int8 column, a non-numeric value must count as failed. | + +| `ImportSummaryCounts.test_failed_rows_are_reported_with_detail` | integration | A count alone is not actionable — the offending row must be named. | + +| `ImportSummaryCounts.test_counts_reconcile_for_a_clean_file` | integration | — | + +| `ImportSummaryCounts.test_counts_reconcile_for_a_mixed_file` | integration | One good row, one in-file duplicate, one uncastable value. | + +| `ImportSummaryCounts.test_counts_are_per_batch_not_cumulative` | integration | A second, larger upload must report its own totals, not a running sum. | + +| `test_all_four_counts_are_returned` | — | — | + +| `test_total_rows_counts_data_rows_not_the_header` | — | — | + +| `test_created_count_matches_rows_actually_inserted` | — | — | + +| `test_duplicate_count_reflects_repeated_rows_in_the_file` | — | Two identical rows plus one distinct: 1 duplicate skipped. | + +| `test_failed_count_reflects_rows_that_could_not_be_cast` | — | With an int8 column, a non-numeric value must count as failed. | + +| `test_failed_rows_are_reported_with_detail` | — | A count alone is not actionable — the offending row must be named. | + +| `test_counts_reconcile_for_a_clean_file` | — | — | + +| `test_counts_reconcile_for_a_mixed_file` | — | One good row, one in-file duplicate, one uncastable value. | + +| `test_counts_are_per_batch_not_cumulative` | — | A second, larger upload must report its own totals, not a running sum. | + +## `tests/test_migrations.py` + +| Test | Markers | Objective | +|---|---|---| +| `MigrationsOnScratchSchema.test_upgrade_head_creates_tables_and_unique_constraints` | integration | — | + +| `MigrationsOnScratchSchema.test_control_schema_is_created_and_removed` | integration | — | + +| `MigrationsOnScratchSchema.test_downgrade_reverses_each_step` | integration | — | + +| `MigrationsOnScratchSchema.test_baseline_adopts_a_fallback_created_schema` | integration | — | + +| `MigrationsOnScratchSchema.test_existing_duplicates_stop_the_upgrade_with_remediation` | integration | — | + +| `RegistryUniqueness.test_bootstrap_brings_the_schema_to_head` | integration | — | + +| `RegistryUniqueness.test_duplicate_hash_is_rejected_by_the_database` | integration | — | + +| `RegistryUniqueness.test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | integration | — | + +| `RegistryUniqueness.test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | integration | — | + +| `test_upgrade_head_creates_tables_and_unique_constraints` | — | — | + +| `test_control_schema_is_created_and_removed` | — | — | + +| `test_downgrade_reverses_each_step` | — | — | + +| `test_baseline_adopts_a_fallback_created_schema` | — | — | + +| `test_existing_duplicates_stop_the_upgrade_with_remediation` | — | — | + +| `test_bootstrap_brings_the_schema_to_head` | — | — | + +| `test_duplicate_hash_is_rejected_by_the_database` | — | — | + +| `test_dynamic_upload_losing_the_race_reports_duplicate_and_commits_nothing` | — | — | + +| `test_te_load_losing_the_race_reports_error_and_rolls_back_rows` | — | — | + +## `tests/test_oidc.py` + +| Test | Markers | Objective | +|---|---|---| +| `TokenVerification.test_valid_token_yields_the_username` | unit | — | + +| `TokenVerification.test_rejected_tokens` | unit | — | + +| `OidcSignIn.test_valid_token_for_a_member_gets_a_session` | integration | — | + +| `OidcSignIn.test_valid_token_but_no_membership_or_account_is_401` | integration | — | + +| `OidcSignIn.test_invalid_token_is_401` | integration | — | + +| `OidcSignIn.test_inactive_user_is_401` | integration | — | + +| `OidcSignIn.test_route_is_404_when_oidc_is_not_configured` | integration | — | + +| `ConfigFailsClosed.test_off_by_default_and_complete_config_passes` | unit | — | + +| `ConfigFailsClosed.test_missing_audience_or_insecure_jwks_url_refuses_to_start` | unit | — | + +| `test_valid_token_yields_the_username` | — | — | + +| `test_rejected_tokens` | — | — | + +| `test_valid_token_for_a_member_gets_a_session` | — | — | + +| `test_valid_token_but_no_membership_or_account_is_401` | — | — | + +| `test_invalid_token_is_401` | — | — | + +| `test_inactive_user_is_401` | — | — | + +| `test_route_is_404_when_oidc_is_not_configured` | — | — | + +| `test_off_by_default_and_complete_config_passes` | — | — | + +| `test_missing_audience_or_insecure_jwks_url_refuses_to_start` | — | — | + +## `tests/test_original_bytes.py` + +| Test | Markers | Objective | +|---|---|---| +| `OriginalBytesAreKept.test_bom_prefixed_csv_round_trips_byte_identical` | integration | — | + +| `OriginalBytesAreKept.test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | integration | — | + +| `OriginalBytesAreKept.test_the_decoded_view_still_feeds_the_loader_and_source_rows` | integration | — | + +| `OriginalBytesAreKept.test_a_failed_upload_keeps_the_raw_original_too` | integration | — | + +| `OriginalBytesAreKept.test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | integration | — | + +| `OriginalBytesAreKept.test_invalid_base64_is_rejected_before_anything_loads` | integration | — | + +| `test_bom_prefixed_csv_round_trips_byte_identical` | — | — | + +| `test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical` | — | — | + +| `test_the_decoded_view_still_feeds_the_loader_and_source_rows` | — | — | + +| `test_a_failed_upload_keeps_the_raw_original_too` | — | — | + +| `test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before` | — | — | + +| `test_invalid_base64_is_rejected_before_anything_loads` | — | — | + +## `tests/test_parity.py` + +| Test | Markers | Objective | +|---|---|---| +| `TestDevSeedCounts.test_dev_seed_row_counts` | — | — | + +| `TestAllEnvironmentsHaveRequiredTables.test_dev_has_required_tables` | — | — | + +| `TestAllEnvironmentsHaveRequiredTables.test_test_has_required_tables` | — | — | + +| `TestAllEnvironmentsHaveRequiredTables.test_staging_has_required_tables` | — | — | + +| `TestAllEnvironmentsHaveRequiredTables.test_prod_has_required_tables` | — | — | + +| `TestNoSeedInStagingAndProd.test_staging_has_no_seed_data` | — | — | + +| `TestNoSeedInStagingAndProd.test_prod_has_no_seed_data` | — | — | + +| `TestIdempotentDeployParity.test_dev_row_counts_stable_after_second_deploy` | — | — | + +| `test_dev_seed_row_counts` | — | — | + +| `test_dev_has_required_tables` | — | — | + +| `test_test_has_required_tables` | — | — | + +| `test_staging_has_required_tables` | — | — | + +| `test_prod_has_required_tables` | — | — | + +| `test_staging_has_no_seed_data` | — | — | + +| `test_prod_has_no_seed_data` | — | — | + +| `test_dev_row_counts_stable_after_second_deploy` | — | — | + +## `tests/test_password_reset.py` + +| Test | Markers | Objective | +|---|---|---| +| `ChangeOwnPassword.test_new_password_works_old_does_not_and_other_sessions_end` | integration | — | + +| `ChangeOwnPassword.test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | integration | — | + +| `ChangeOwnPassword.test_short_new_password_is_rejected` | integration | — | + +| `ChangeOwnPassword.test_needs_a_session` | integration | — | + +| `AdminReset.test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | integration | — | + +| `AdminReset.test_reset_is_written_to_the_tenants_audit_log` | integration | — | + +| `AdminReset.test_non_admin_cannot_reset` | integration | — | + +| `AdminReset.test_admin_cannot_reset_a_user_in_another_tenant` | integration | — | + +| `AdminReset.test_user_who_is_also_in_another_tenant_is_refused` | integration | — | + +| `BreakGlassReset.test_resets_password_clears_lockout_and_ends_sessions` | integration | — | + +| `BreakGlassReset.test_unknown_user_and_short_password_fail_without_changes` | integration | — | + +| `test_new_password_works_old_does_not_and_other_sessions_end` | — | — | + +| `test_wrong_current_password_is_400_changes_nothing_and_counts_as_a_failure` | — | — | + +| `test_short_new_password_is_rejected` | — | — | + +| `test_needs_a_session` | — | — | + +| `test_admin_reset_sets_password_unlocks_and_ends_the_users_sessions` | — | — | + +| `test_reset_is_written_to_the_tenants_audit_log` | — | — | + +| `test_non_admin_cannot_reset` | — | — | + +| `test_admin_cannot_reset_a_user_in_another_tenant` | — | — | + +| `test_user_who_is_also_in_another_tenant_is_refused` | — | — | + +| `test_resets_password_clears_lockout_and_ends_sessions` | — | — | + +| `test_unknown_user_and_short_password_fail_without_changes` | — | — | + +## `tests/test_preflight_ps1.py` + +| Test | Markers | Objective | +|---|---|---| +| `test_no_null_coalescing_operator` | regression, unit | `??` / `??=` is PowerShell 7+ only and fails to parse on 5.1. | + +| `test_no_other_powershell7_only_syntax` | regression, unit | — | + +| `test_no_utf8_bom` | regression, unit | A BOM caused a noisy line-1 diff during the fix; keep the file BOM-free. | + +| `test_port_label_is_not_hardcoded` | regression, unit | The check label must follow the configured port (was fixed at 'port 5432'). | + +| `test_connection_defaults_resolved_once_and_reused` | regression, unit | Defaults live in variables; connection commands must use them. | + +| `test_default_port_and_host_unchanged` | regression, unit | — | + +| `test_exit_code_contract_documented_and_implemented` | regression, unit | — | + +| `test_parses_under_windows_powershell` | regression | Parse with the real PowerShell parser (5.1 on stock Windows). | + +| `test_runs_with_skippg_and_reports_summary` | regression | — | + +| `test_default_port_label_when_pgport_unset` | regression | — | + +| `test_port_label_follows_pgport` | regression | — | + +| `test_pg_checks_are_optional_never_fail` | regression | — | + +## `tests/test_preserved_uploads.py` + +| Test | Markers | Objective | +|---|---|---| +| `PreservedUploads.test_original_bytes_and_rows_are_kept_with_the_outcome` | integration | — | + +| `PreservedUploads.test_a_failed_upload_still_keeps_its_original` | integration | — | + +| `PreservedUploads.test_failure_reasons_and_log_are_readable_from_another_session` | integration | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | + +| `PreservedUploads.test_partly_failed_and_duplicate_outcomes` | integration | — | + +| `PreservedUploads.test_overwrite_keeps_the_replaced_original` | integration | — | + +| `PreservedUploads.test_preserved_tables_reject_update_and_delete` | integration | — | + +| `PreservedUploads.test_unknown_upload_is_404` | integration | — | + +| `test_original_bytes_and_rows_are_kept_with_the_outcome` | — | — | + +| `test_a_failed_upload_still_keeps_its_original` | — | — | + +| `test_failure_reasons_and_log_are_readable_from_another_session` | — | M3c (#107): the reasons rows failed live on the server, not only in the uploading browser. | + +| `test_partly_failed_and_duplicate_outcomes` | — | — | + +| `test_overwrite_keeps_the_replaced_original` | — | — | + +| `test_preserved_tables_reject_update_and_delete` | — | — | + +| `test_unknown_upload_is_404` | — | — | + +## `tests/test_ragged_rows.py` + +| Test | Markers | Objective | +|---|---|---| +| `ExtraCells.test_non_empty_cells_beyond_the_header_are_returned` | unit | — | + +| `ExtraCells.test_trailing_empty_cells_are_not_extra` | unit | — | + +| `ExtraCells.test_short_rows_have_no_extra_cells` | unit | — | + +| `RaggedRowsOverTheApi.test_dynamic_mode_reports_a_row_wider_than_the_header` | integration | — | + +| `RaggedRowsOverTheApi.test_dynamic_mode_accepts_a_trailing_empty_cell` | integration | — | + +| `RaggedRowsOverTheApi.test_te_mode_reports_a_row_wider_than_the_header` | integration | — | + +| `RaggedRowsOverTheApi.test_multiple_ragged_rows_each_get_a_reason` | integration | Every ragged row gets its own entry in rowErrors with a reason. | + +| `test_non_empty_cells_beyond_the_header_are_returned` | — | — | + +| `test_trailing_empty_cells_are_not_extra` | — | — | + +| `test_short_rows_have_no_extra_cells` | — | — | + +| `test_dynamic_mode_reports_a_row_wider_than_the_header` | — | — | + +| `test_dynamic_mode_accepts_a_trailing_empty_cell` | — | — | + +| `test_te_mode_reports_a_row_wider_than_the_header` | — | — | + +| `test_multiple_ragged_rows_each_get_a_reason` | — | Every ragged row gets its own entry in rowErrors with a reason. | + +## `tests/test_registry_race.py` + +| Test | Markers | Objective | +|---|---|---| +| `RegistryRace.test_concurrent_identical_uploads_one_wins` | integration | Two threads upload the same content simultaneously. | + +| `RegistryRace.test_concurrent_same_name_different_content` | integration | Two threads upload different content under the same filename. | + +| `test_concurrent_identical_uploads_one_wins` | — | Two threads upload the same content simultaneously. | + +| `test_concurrent_same_name_different_content` | — | Two threads upload different content under the same filename. | + +## `tests/test_regression.py` + +| Test | Markers | Objective | +|---|---|---| +| `RegressionBomHandling.test_bom_csv_is_accepted` | — | — | + +| `RegressionCrlfLineEndings.test_crlf_csv_accepted` | — | — | + +| `RegressionHeaderOnlyCsv.test_header_only_exits_cleanly` | — | — | + +| `RegressionSingleColumnCsv.test_single_column_valid_rows_pass` | — | — | + +| `RegressionNoTraceback.test_missing_env_vars_no_traceback` | — | — | + +| `RegressionNoTraceback.test_nonexistent_file_no_traceback` | — | — | + +| `RegressionNoTraceback.test_invalid_utf8_no_traceback` | — | — | + +| `test_bom_csv_is_accepted` | — | — | + +| `test_crlf_csv_accepted` | — | — | + +| `test_header_only_exits_cleanly` | — | — | + +| `test_single_column_valid_rows_pass` | — | — | + +| `test_missing_env_vars_no_traceback` | — | — | + +| `test_nonexistent_file_no_traceback` | — | — | + +| `test_invalid_utf8_no_traceback` | — | — | + +## `tests/test_review.py` + +| Test | Markers | Objective | +|---|---|---| +| `Review.test_pages_through_every_row_with_validation` | integration | — | + +| `Review.test_validates_rows_beyond_the_reported_error_cap` | integration | — | + +| `Review.test_row_decisions_record_actor_and_reason_and_the_latest_wins` | integration | — | + +| `Review.test_cell_edits_are_versions_over_the_unchanged_original` | integration | — | + +| `Review.test_refusals` | integration | — | + +| `Review.test_te_review_shows_the_database_reason_a_row_was_rejected` | integration | — | + +| `Review.test_changes_are_audited_and_insert_only` | integration | — | + +| `test_pages_through_every_row_with_validation` | — | — | + +| `test_validates_rows_beyond_the_reported_error_cap` | — | — | + +| `test_row_decisions_record_actor_and_reason_and_the_latest_wins` | — | — | + +| `test_cell_edits_are_versions_over_the_unchanged_original` | — | — | + +| `test_refusals` | — | — | + +| `test_te_review_shows_the_database_reason_a_row_was_rejected` | — | — | + +| `test_changes_are_audited_and_insert_only` | — | — | + +## `tests/test_review_cases.py` + +| Test | Markers | Objective | +|---|---|---| +| `ReviewCases.test_P02_paging_bounds` | integration | — | + +| `ReviewCases.test_P04_ragged_row_is_flagged` | integration | — | + +| `ReviewCases.test_P05_a_failed_upload_can_be_reviewed` | integration | — | + +| `ReviewCases.test_P08_an_edit_to_a_bad_value_is_flagged` | integration | — | + +| `ReviewCases.test_P09_edit_by_column_index_and_to_an_empty_value` | integration | — | + +| `ReviewCases.test_P10_reason_is_stored_trimmed` | integration | — | + +| `ReviewCases.test_P11_actor_is_the_signed_in_user` | integration | — | + +| `ReviewCases.test_P12_every_role_can_view_rows_and_history` | integration | — | + +| `ReviewCases.test_N05_row_numbers_outside_the_upload` | integration | — | + +| `ReviewCases.test_N06_column_index_out_of_range` | integration | — | + +| `ReviewCases.test_N07_unknown_upload_on_every_route` | integration | — | + +| `ReviewCases.test_N08_N09_uploader_and_auditor_cannot_change` | integration | — | + +| `ReviewCases.test_N10_an_invalid_session_is_refused` | integration | — | + +| `ReviewCases.test_N11_another_tenant_cannot_see_or_change_the_upload` | integration | — | + +| `ReviewCases.test_N12_missing_or_wrong_body_fields` | integration | — | + +| `ReviewCases.test_N13_review_tables_reject_update_delete_and_truncate` | integration | — | + +| `ReviewCases.test_N14_database_rejects_a_blank_reason` | integration | — | + +| `ReviewCases.test_N16_a_non_numeric_upload_id` | integration | — | + +| `test_P02_paging_bounds` | — | — | + +| `test_P04_ragged_row_is_flagged` | — | — | + +| `test_P05_a_failed_upload_can_be_reviewed` | — | — | + +| `test_P08_an_edit_to_a_bad_value_is_flagged` | — | — | + +| `test_P09_edit_by_column_index_and_to_an_empty_value` | — | — | + +| `test_P10_reason_is_stored_trimmed` | — | — | + +| `test_P11_actor_is_the_signed_in_user` | — | — | + +| `test_P12_every_role_can_view_rows_and_history` | — | — | + +| `test_N05_row_numbers_outside_the_upload` | — | — | + +| `test_N06_column_index_out_of_range` | — | — | + +| `test_N07_unknown_upload_on_every_route` | — | — | + +| `test_N08_N09_uploader_and_auditor_cannot_change` | — | — | + +| `test_N10_an_invalid_session_is_refused` | — | — | + +| `test_N11_another_tenant_cannot_see_or_change_the_upload` | — | — | + +| `test_N12_missing_or_wrong_body_fields` | — | — | + +| `test_N13_review_tables_reject_update_delete_and_truncate` | — | — | + +| `test_N14_database_rejects_a_blank_reason` | — | — | + +| `test_N16_a_non_numeric_upload_id` | — | — | + +## `tests/test_roles.py` + +| Test | Markers | Objective | +|---|---|---| +| `MatrixIsDocumented.test_docs_table_matches_the_enforced_permissions` | unit | — | + +| `MatrixIsDocumented.test_every_action_has_a_route_and_only_known_roles` | unit | — | + +| `RoleMatrixOverTheApi.test_every_role_against_every_route` | integration | — | + +| `RoleMatrixOverTheApi.test_no_session_requests_are_not_role_checked` | integration | — | + +| `test_docs_table_matches_the_enforced_permissions` | — | — | + +| `test_every_action_has_a_route_and_only_known_roles` | — | — | + +| `test_every_role_against_every_route` | — | — | + +| `test_no_session_requests_are_not_role_checked` | — | — | + +## `tests/test_runtime_role.py` + +| Test | Markers | Objective | +|---|---|---| +| `RuntimeRole.test_script_keeps_the_password_out_of_its_output` | integration | — | + +| `RuntimeRole.test_the_api_works_as_the_runtime_role` | integration | — | + +| `RuntimeRole.test_runtime_role_cannot_change_or_unprotect_append_only_tables` | integration | — | + +| `test_script_keeps_the_password_out_of_its_output` | — | — | + +| `test_the_api_works_as_the_runtime_role` | — | — | + +| `test_runtime_role_cannot_change_or_unprotect_append_only_tables` | — | — | + +## `tests/test_security.py` + +| Test | Markers | Objective | +|---|---|---| +| `TestNoHardcodedCredentials.test_validator_no_hardcoded_credentials` | — | — | + +| `TestNoHardcodedCredentials.test_runner_no_hardcoded_credentials` | — | — | + +| `TestNoHardcodedCredentials.test_gap_report_no_hardcoded_credentials` | — | — | + +| `TestNoFStringSql.test_validator_no_fstring_sql` | — | — | + +| `TestNoFStringSql.test_runner_no_fstring_sql` | — | — | + +| `TestConfigEnvExample.test_pg_password_is_empty` | — | — | + +| `TestShellScriptsNoPrintedPasswords.test_no_echo_password_in_adapters` | — | — | + +| `TestShellScriptsNoPrintedPasswords.test_no_echo_password_in_loaders` | — | — | + +| `TestSqlFilesNoInlineCredentials.test_env_sql_files_no_hardcoded_passwords` | — | — | + +| `test_validator_no_hardcoded_credentials` | — | — | + +| `test_runner_no_hardcoded_credentials` | — | — | + +| `test_gap_report_no_hardcoded_credentials` | — | — | + +| `test_validator_no_fstring_sql` | — | — | + +| `test_runner_no_fstring_sql` | — | — | + +| `test_pg_password_is_empty` | — | — | + +| `test_no_echo_password_in_adapters` | — | — | + +| `test_no_echo_password_in_loaders` | — | — | + +| `test_env_sql_files_no_hardcoded_passwords` | — | — | + +## `tests/test_service_loader.py` + +| Test | Markers | Objective | +|---|---|---| +| `ServiceLoaderUnitTests.test_dynamic_upload_rejects_mismatched_type_list_length` | — | — | + +| `ServiceLoaderUnitTests.test_te_cleanup_prior_registry_entries_drops_dynamic_table` | — | — | + +| `test_dynamic_upload_rejects_mismatched_type_list_length` | — | — | + +| `test_te_cleanup_prior_registry_entries_drops_dynamic_table` | — | — | + +## `tests/test_snapshot.py` + +| Test | Markers | Objective | +|---|---|---| +| `SnapshotBasicMixedInput.test_valid_csv_matches_snapshot` | — | — | + +| `SnapshotBasicMixedInput.test_skip_csv_matches_snapshot` | — | — | + +| `SnapshotAllValidInput.test_all_valid_no_skips` | — | — | + +| `test_valid_csv_matches_snapshot` | — | — | + +| `test_skip_csv_matches_snapshot` | — | — | + +| `test_all_valid_no_skips` | — | — | + +## `tests/test_te_loader_integration.py` + +| Test | Markers | Objective | +|---|---|---| +| `TeUploadOrganisations.test_valid_te_upload_returns_ok` | integration | A well-formed T&E CSV must return status=ok. | + +| `TeUploadOrganisations.test_inserted_rows_count_matches_data_rows` | integration | insertedRows must equal the number of data rows supplied. | + +| `TeUploadOrganisations.test_table_name_in_response_is_schema_qualified` | integration | tableName must be schema-qualified (e.g. te_dev.organisations). | + +| `TeUploadOrganisations.test_rows_are_queryable_after_upload` | integration | Inserted rows must appear in the T&E table row count. | + +| `TeUploadOrganisations.test_core_te_tables_undisturbed_after_upload` | integration | A T&E upload must not drop or alter the other 11 core tables. | + +| `TeUploadOrganisations.test_reupload_cleans_prior_registry_entry` | integration | Re-uploading the same file name must not duplicate registry entries. | + +| `TeUploadOrganisations.test_invalid_target_table_is_rejected` | integration | A targetTable not in TE_TABLES must return status=error. | + +| `TeUploadOrganisations.test_empty_file_is_rejected` | integration | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | + +| `TeUploadOrganisations.test_header_only_file_is_rejected` | integration | A header-only CSV (no data rows) must return status=invalid_structure. | + +| `TeUploadOrganisations.test_unknown_columns_are_rejected` | integration | Columns not in the target table must return status=error. | + +| `TeUploadOrganisations.test_missing_target_table_param_is_rejected` | integration | mode=te with no targetTable must return 422 from the API layer. | + +| `DynamicOverwriteOfTeOriginFile.test_dynamic_overwrite_of_te_origin_entry_succeeds` | integration | A dynamic-mode overwrite of a filename previously registered in | + +| `test_valid_te_upload_returns_ok` | — | A well-formed T&E CSV must return status=ok. | + +| `test_inserted_rows_count_matches_data_rows` | — | insertedRows must equal the number of data rows supplied. | + +| `test_table_name_in_response_is_schema_qualified` | — | tableName must be schema-qualified (e.g. te_dev.organisations). | + +| `test_rows_are_queryable_after_upload` | — | Inserted rows must appear in the T&E table row count. | + +| `test_core_te_tables_undisturbed_after_upload` | — | A T&E upload must not drop or alter the other 11 core tables. | + +| `test_reupload_cleans_prior_registry_entry` | — | Re-uploading the same file name must not duplicate registry entries. | + +| `test_invalid_target_table_is_rejected` | — | A targetTable not in TE_TABLES must return status=error. | + +| `test_empty_file_is_rejected` | — | Empty CSV is rejected — 422 from Pydantic min_length or status=error. | + +| `test_header_only_file_is_rejected` | — | A header-only CSV (no data rows) must return status=invalid_structure. | + +| `test_unknown_columns_are_rejected` | — | Columns not in the target table must return status=error. | + +| `test_missing_target_table_param_is_rejected` | — | mode=te with no targetTable must return 422 from the API layer. | + +| `test_dynamic_overwrite_of_te_origin_entry_succeeds` | — | A dynamic-mode overwrite of a filename previously registered in | + +## `tests/test_tenant_isolation.py` + +| Test | Markers | Objective | +|---|---|---| +| `CrossTenantIsolation.test_upload_lands_in_the_tenants_own_schema` | integration | — | + +| `CrossTenantIsolation.test_other_tenant_cannot_list_read_or_delete` | integration | — | + +| `CrossTenantIsolation.test_requests_without_a_session_see_no_tenant_data` | integration | — | + +| `CrossTenantIsolation.test_duplicate_detection_is_per_tenant` | integration | — | + +| `CrossTenantIsolation.test_audit_log_is_per_tenant` | integration | — | + +| `CrossTenantIsolation.test_te_mode_loads_into_the_tenants_own_te_tables` | integration | — | + +| `CrossTenantIsolation.test_te_table_counts_are_per_tenant` | integration | — | + +| `CrossTenantIsolation.test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | integration | — | + +| `CrossTenantIsolation.test_invalid_session_is_401_not_a_fallback_to_default` | integration | — | + +| `CrossTenantIsolation.test_tenant_schema_is_at_head_with_its_own_registry` | integration | — | + +| `CrossTenantIsolation.test_concurrent_requests_never_see_another_tenants_schema` | integration | — | + +| `CrossTenantIsolation.test_provisioning_is_idempotent` | integration | — | + +| `test_upload_lands_in_the_tenants_own_schema` | — | — | + +| `test_other_tenant_cannot_list_read_or_delete` | — | — | + +| `test_requests_without_a_session_see_no_tenant_data` | — | — | + +| `test_duplicate_detection_is_per_tenant` | — | — | + +| `test_audit_log_is_per_tenant` | — | — | + +| `test_te_mode_loads_into_the_tenants_own_te_tables` | — | — | + +| `test_te_table_counts_are_per_tenant` | — | — | + +| `test_tenant_te_tables_keep_their_foreign_keys_and_triggers` | — | — | + +| `test_invalid_session_is_401_not_a_fallback_to_default` | — | — | + +| `test_tenant_schema_is_at_head_with_its_own_registry` | — | — | + +| `test_concurrent_requests_never_see_another_tenants_schema` | — | — | + +| `test_provisioning_is_idempotent` | — | — | + diff --git a/docs/prompts/ART_FRAMEWORK.md b/docs/prompts/ART_FRAMEWORK.md index 3f5c074..1d54bba 100644 --- a/docs/prompts/ART_FRAMEWORK.md +++ b/docs/prompts/ART_FRAMEWORK.md @@ -237,7 +237,7 @@ Follow the repository's no-skip policy: a missing prerequisite fails with remedi ### Evidence rules - A requirement is "Verified" only when a test run proves it. Quote the actual command and the actual pass, fail and skip counts. -- Documents such as `VCRM.md`, `FIXES_APPLIED.md` and `API_INTEGRATION.md` are claims, not evidence. `API_INTEGRATION.md` describes a superseded API. +- Documents such as `VCRM.md`, `FIXES_APPLIED.md` and `API_INTEGRATION.md` are claims, not evidence. `API_INTEGRATION.md` does not list routes; the live list is the OpenAPI schema at `/docs` (`/openapi.json`) on a running API. - If something could not be run (for example, no database available), say so plainly. - When searching the repository, exclude the stale worktree `.claude/worktrees/velvety-chasing-badger/`. It holds an old copy of `api/`. diff --git a/docs/test-cases/M4-review.md b/docs/test-cases/M4-review.md index bb5a060..ee4a634 100644 --- a/docs/test-cases/M4-review.md +++ b/docs/test-cases/M4-review.md @@ -88,7 +88,7 @@ $u = $up.uploadId ## Not covered here - **The review screen** (M4b, PR #143) has its own component tests (`review-table.test.tsx`) and the browser check `scripts/e2e/m4-review.js`. -- **T&E-mode uploads** store no column types, so their values are reviewed as text with no type checks. That's a known limit, noted on #142. +- **T&E-mode uploads** store the target table's column types (#190), and review shows the database's reason for rows it rejected (CHECK, FK, UNIQUE) until the row is edited. Edited T&E rows are re-checked against types only; constraints are checked at M6's import. Uploads made before #190 have no stored types and review as text. - **Applying review to the import** belongs to M6 (#110). It isn't tested here, because it isn't built yet. ## Record of runs diff --git a/evals/runner.py b/evals/runner.py index 5a9c0a0..ffffc25 100644 --- a/evals/runner.py +++ b/evals/runner.py @@ -358,7 +358,7 @@ def _run_deploy_dev_twice( return result env = _pg_env() - psql_args = ["psql", "-f", str(env_dev_sql)] + psql_args = ["psql", "-v", "ON_ERROR_STOP=1", "-f", str(env_dev_sql)] r1 = subprocess.run(psql_args, env=env, capture_output=True, text=True, timeout=120) counts_1 = _count_dev_rows() diff --git a/scripts/check_coverage_floors.py b/scripts/check_coverage_floors.py new file mode 100755 index 0000000..9e74a5e --- /dev/null +++ b/scripts/check_coverage_floors.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Check per-file coverage floors for api/services loaders (#194). + +Usage: + python3 scripts/check_coverage_floors.py coverage-services.json + +Reads the JSON coverage report produced by pytest-cov and verifies that +each loader file meets its minimum coverage percentage. Exits non-zero +if any file drops below its floor. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +FLOORS: dict[str, int] = { + "api/services/csv_parse.py": 79, + "api/services/dynamic_loader.py": 84, + "api/services/te_loader.py": 96, +} + +GREEN = "\033[0;32m" +RED = "\033[0;31m" +NC = "\033[0m" + + +def main() -> int: + if len(sys.argv) < 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + return 2 + + report_path = Path(sys.argv[1]) + if not report_path.exists(): + print(f"{RED}Coverage report not found: {report_path}{NC}", file=sys.stderr) + return 1 + + with open(report_path) as f: + data = json.load(f) + + files = data.get("files", {}) + failed = False + + print("\n api/services coverage floors (#194)") + print(" " + "─" * 56) + print(f" {'File':<40} {'Actual':>7} {'Floor':>7} Result") + print(" " + "─" * 56) + + for filepath, floor in sorted(FLOORS.items()): + info = files.get(filepath) + if info is None: + print(f" {filepath:<40} {'n/a':>7} {floor:>6}% {RED}MISSING{NC}") + failed = True + continue + + pct = info["summary"]["percent_covered"] + ok = pct >= floor + tag = f"{GREEN}PASS{NC}" if ok else f"{RED}FAIL{NC}" + print(f" {filepath:<40} {pct:>6.1f}% {floor:>6}% {tag}") + if not ok: + failed = True + + print(" " + "─" * 56) + + if failed: + print(f"\n {RED}Coverage gate FAILED — one or more loaders below floor.{NC}\n") + return 1 + else: + print(f"\n {GREEN}Coverage gate passed.{NC}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_catalogue.py b/scripts/test_catalogue.py new file mode 100755 index 0000000..d25a0e5 --- /dev/null +++ b/scripts/test_catalogue.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Generate docs/TEST_CASES.md from the test tree (#169). + +Usage: + python3 scripts/test_catalogue.py # generate docs/TEST_CASES.md + python3 scripts/test_catalogue.py --check # CI mode: fail if doc is stale + +Walks every *.py file under tests/, extracts test functions/methods with +their docstrings and pytest markers, and produces a Markdown table grouped +by file. The --check flag compares the freshly generated content against +the committed file and exits non-zero if they differ (with a diff). +""" +from __future__ import annotations + +import argparse +import ast +import difflib +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TESTS_DIR = ROOT / "tests" +OUTPUT = ROOT / "docs" / "TEST_CASES.md" + +MARKER_NAMES = {"unit", "integration", "e2e", "regression", "security", + "snapshot", "parity"} + + +def _extract_markers(decorators: list[ast.expr]) -> list[str]: + """Return pytest marker names from decorator nodes.""" + markers: list[str] = [] + for dec in decorators: + if isinstance(dec, ast.Attribute) and isinstance(dec.value, ast.Attribute): + if (isinstance(dec.value.value, ast.Name) + and dec.value.value.id == "pytest" + and dec.value.attr == "mark"): + markers.append(dec.attr) + elif isinstance(dec, ast.Call) and isinstance(dec.func, ast.Attribute): + attr = dec.func + if isinstance(attr.value, ast.Attribute): + if (isinstance(attr.value.value, ast.Name) + and attr.value.value.id == "pytest" + and attr.value.attr == "mark"): + markers.append(attr.attr) + return markers + + +def _first_line_of_docstring(node: ast.AST) -> str: + """Return the first non-empty line of the docstring, or ''.""" + ds = ast.get_docstring(node) + if not ds: + return "" + for line in ds.splitlines(): + stripped = line.strip() + if stripped: + return stripped + return "" + + +def _collect_tests(filepath: Path) -> list[dict]: + """Parse one test file and return a list of test dicts.""" + try: + tree = ast.parse(filepath.read_text(encoding="utf-8"), filename=str(filepath)) + except SyntaxError: + return [] + + tests: list[dict] = [] + rel = filepath.relative_to(ROOT) + + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + class_markers = _extract_markers(node.decorator_list) + for item in node.body: + if isinstance(item, ast.FunctionDef) and item.name.startswith("test"): + markers = class_markers + _extract_markers(item.decorator_list) + tests.append({ + "file": str(rel), + "class": node.name, + "name": item.name, + "line": item.lineno, + "markers": sorted(set(m for m in markers if m in MARKER_NAMES)), + "objective": _first_line_of_docstring(item), + }) + elif isinstance(item := node, ast.FunctionDef) and item.name.startswith("test"): + markers = _extract_markers(item.decorator_list) + tests.append({ + "file": str(rel), + "class": "", + "name": item.name, + "line": item.lineno, + "markers": sorted(set(m for m in markers if m in MARKER_NAMES)), + "objective": _first_line_of_docstring(item), + }) + + return tests + + +def generate() -> str: + """Generate the full TEST_CASES.md content.""" + all_tests: list[dict] = [] + for py in sorted(TESTS_DIR.rglob("test_*.py")): + all_tests.extend(_collect_tests(py)) + + lines: list[str] = [] + lines.append("# Test Catalogue") + lines.append("") + lines.append("> Auto-generated by `scripts/test_catalogue.py` — do not edit by hand.") + lines.append(f"> {len(all_tests)} tests across {len(set(t['file'] for t in all_tests))} files.") + lines.append("") + lines.append("To regenerate: `python3 scripts/test_catalogue.py`") + lines.append("") + + # Group by file + current_file = "" + for t in all_tests: + if t["file"] != current_file: + current_file = t["file"] + lines.append(f"## `{current_file}`") + lines.append("") + lines.append("| Test | Markers | Objective |") + lines.append("|---|---|---|") + + markers_str = ", ".join(t["markers"]) if t["markers"] else "—" + name = f"`{t['class']}.{t['name']}`" if t["class"] else f"`{t['name']}`" + obj = t["objective"] or "—" + lines.append(f"| {name} | {markers_str} | {obj} |") + lines.append("") + + lines.append("") + return "\n".join(lines) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", + help="CI mode: exit 1 if docs/TEST_CASES.md is stale") + args = parser.parse_args() + + fresh = generate() + + if args.check: + if not OUTPUT.exists(): + print("docs/TEST_CASES.md does not exist. Run:\n" + " python3 scripts/test_catalogue.py\n" + "and commit the result.", file=sys.stderr) + return 1 + + existing = OUTPUT.read_text(encoding="utf-8") + if existing == fresh: + print("docs/TEST_CASES.md is up to date.") + return 0 + + diff = difflib.unified_diff( + existing.splitlines(keepends=True), + fresh.splitlines(keepends=True), + fromfile="docs/TEST_CASES.md (committed)", + tofile="docs/TEST_CASES.md (generated)", + ) + sys.stderr.writelines(diff) + print("\ndocs/TEST_CASES.md is stale. Run:\n" + " python3 scripts/test_catalogue.py\n" + "and commit the result.", file=sys.stderr) + return 1 + + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + OUTPUT.write_text(fresh, encoding="utf-8") + count = fresh.count("| `") + print(f"Wrote docs/TEST_CASES.md ({count} tests)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/spikes/streamlit_review/README.md b/spikes/streamlit_review/README.md new file mode 100644 index 0000000..69f900b --- /dev/null +++ b/spikes/streamlit_review/README.md @@ -0,0 +1,66 @@ +# #154 spike: M4 review screen in Streamlit + +Throwaway learning code. The React screen (`csv-table-hub-main/src/components/review-table.tsx`) stays live. Nothing here is wired into CI, the build or the offline bundle. + +## Run it + +With the API on `localhost:8000` (same as frontend dev): + +```bash +pip install -r spikes/streamlit_review/requirements.txt +``` + +```bash +streamlit run spikes/streamlit_review/app.py +``` + +Sign in with a tenant account, or leave the sidebar empty in dev to use the shared API key (`API_KEY` env var, no role checks). Set `API_URL` if the API isn't on `localhost:8000`. + +## Scope check (issue #154) + +| Must support | Result | +|---|---| +| Paging (100 rows) | Page number input. Rows come from `GET /uploads/{id}/rows` | +| Cell edit dialog with reason | `st.dialog`. Save is disabled until a reason is entered (but see strain 2) | +| Exclude / include | `st.dialog` with a required reason | +| History view | `st.dialog`, listing who / when / what / why | +| Role gating | Uploader and auditor see the grid, errors and history only. Reviewer and tenant_admin also get Edit / Exclude | +| Red invalid / blue edited tint | pandas `Styler` backgrounds. Excluded rows are greyed out | +| No new backend code | None. Only existing endpoints are used | + +All of it in one ~180-line Python file, with no build step and no Node. Checked end to end with Playwright against a local API (reviewer: edit, exclude and history round-trip; uploader: read-only). + +## What was comfortable + +- **Every required behaviour worked first try** on standard widgets: `st.dataframe` row selection, `st.dialog`, and `Styler`. +- **Role gating is a plain `if`.** +- **No browser storage.** `st.session_state` lives on the Streamlit server, per tab, and sign-out clears all of it. The class of leak in #176 (tenant A's history in tenant B's `localStorage`) can't happen with this model. +- **One language.** The screen reads like the API code next to it. + +## Where it strained + +1. **No click-to-edit on a cell.** `st.dataframe` only reports row and column selection, not cell clicks. Editing takes three steps: select the row, click "Edit a cell", pick the column. In React you click the cell. `st.data_editor` edits inline, but it has no hook for a per-edit "reason" dialog. +2. **Save enables on Enter or blur, not on each keystroke.** Streamlit only sends a text input's value when the field is committed. Type a reason and Save stays disabled until you press Enter or Tab. The API still rejects an empty reason (422), so this is a UX issue, not a correctness one. +3. **No per-cell tooltips.** Streamlit doesn't render `Styler` tooltips, so "original value" moved to the row detail panel under the grid. +4. **The whole script reruns on every click.** Each interaction refetches the uploads list and the 100-row page. That's fine locally at 100 rows. It's the model to keep in mind as screens grow. +5. **Styling is limited to colours and backgrounds.** CSS `opacity` is ignored, which is why excluded rows use grey text. The overall look is Streamlit's, not the app's. +6. **It's a new server to deploy:** a Streamlit process (port 8501, WebSocket) next to the API. For air-gapped installs (#60), the wheel ships its pre-built frontend (no Node at build time, unlike Reflex). Set `browser.gatherUsageStats = false`. Offline packaging wasn't tested in this spike. + +## Outcome + +Choosing the outcome is the product owner's call after trying it (issue #154, "Done when"). + +My read: strains 1–3 are all on the reviewer's editing path. +- If reviewers mostly look and occasionally fix a cell, Streamlit is fine: **outcome 1**. +- If heavy cell-by-cell editing is the main job, click-to-edit matters, which points to **outcome 2**: NiceGUI, whose AG Grid exposes cell-click events, still in Python. + +## What each outcome means for the parked issues + +| Issue | Outcome 1/2 (Python-native) | Outcome 3 (keep React) | +|---|---|---| +| #176 localStorage leak | Moot once the screen is replaced (server-side session state) | Server-backed, tenant-scoped history | +| #179 lint baseline | Moot with the React code | `npm run lint -- --fix`, then add to CI | +| #186 header mapping | API side either way. The UI control is rebuilt or dropped | Implement or remove the control | +| #187 failed uploads unreachable | The upload picker here already lists every attempt, failed ones included | Patch the browser queue and the response schema | +| #188 misleading Cancel | Don't build a Cancel without server-side cancellation | Relabel now | +| #190 T&E errors | Backend fix (PR #203) applies to both | diff --git a/spikes/streamlit_review/app.py b/spikes/streamlit_review/app.py new file mode 100644 index 0000000..66c3433 --- /dev/null +++ b/spikes/streamlit_review/app.py @@ -0,0 +1,183 @@ +"""#154 spike: the M4 review screen in Streamlit, against the existing FastAPI API. + +Throwaway learning code, not production. The React screen +(csv-table-hub-main/src/components/review-table.tsx) stays live. + + pip install -r spikes/streamlit_review/requirements.txt + streamlit run spikes/streamlit_review/app.py + +API_URL (default http://localhost:8000) and API_KEY (dev, no sign-in) come from the environment. +""" + +from __future__ import annotations + +import os + +import pandas as pd +import requests +import streamlit as st + +API = os.environ.get("API_URL", "http://localhost:8000").rstrip("/") +PAGE = 100 +REVIEW_ROLES = {"reviewer", "tenant_admin"} # api/auth.py PERMISSIONS["review_rows"] + +st.set_page_config(page_title="Review", layout="wide") +ss = st.session_state # per browser tab, held server-side: nothing lands in localStorage + + +def call(method: str, path: str, **kw): + headers = {"Authorization": f"Bearer {ss.session['token']}"} if "session" in ss else \ + {"X-API-Key": os.environ.get("API_KEY", "")} + r = requests.request(method, API + path, headers=headers, timeout=30, **kw) + if r.status_code == 401 and "session" in ss: + del ss["session"] + st.rerun() + if not r.ok: + raise RuntimeError(r.json().get("detail", r.text) if r.headers.get("content-type", "").startswith( + "application/json") else r.text) + return r.json() + + +# ── Sign-in ──────────────────────────────────────────────────────────────────── + +def sign_in(): + with st.sidebar.form("login"): + tenant = st.text_input("Tenant") + username = st.text_input("Username") + password = st.text_input("Password", type="password") + if st.form_submit_button("Sign in"): + r = requests.post(f"{API}/api/auth/login", timeout=30, + json={"tenant": tenant, "username": username, "password": password}) + if r.ok: + ss.clear() + ss.session = r.json() + st.rerun() + st.error("Invalid tenant, username or password") + + +if "session" in ss: + s = ss.session + st.sidebar.write(f"**{s['username']}** · {s['tenant']} · `{s['role']}`") + if st.sidebar.button("Sign out"): + try: + call("POST", "/api/auth/logout") + except RuntimeError: + pass + ss.clear() # all state, so the next tenant sees nothing of this one's (cf. #176) + st.rerun() + can_review = s["role"] in REVIEW_ROLES +else: + sign_in() + st.sidebar.caption("Or, in dev, carry on with the shared API key (no role checks).") + can_review = True + + +# ── Dialogs ──────────────────────────────────────────────────────────────────── + +def _save(fn) -> None: + try: + fn() + except RuntimeError as exc: + st.error(str(exc)) + return + st.rerun() + + +@st.dialog("Edit cell") +def edit_dialog(upload_id: str, row: dict, columns: list[str]): + col = st.selectbox("Column", range(len(columns)), format_func=lambda i: columns[i]) + st.caption(f"Original: `{row['original'][col]}` · Current: `{row['current'][col]}`") + value = st.text_input("New value", value=row["current"][col], key=f"v{row['rowNumber']}_{col}") + reason = st.text_input("Reason (required)") + if st.button("Save", type="primary", disabled=not reason.strip()): + _save(lambda: call("PUT", f"/api/csv/uploads/{upload_id}/rows/{row['rowNumber']}/cells/{col}", + json={"value": value, "reason": reason})) + + +@st.dialog("Change decision") +def decision_dialog(upload_id: str, row: dict, decision: str): + st.write(f"**{decision.title()}** row {row['rowNumber']}") + reason = st.text_input("Reason (required)") + if st.button("Save", type="primary", disabled=not reason.strip()): + _save(lambda: call("PUT", f"/api/csv/uploads/{upload_id}/rows/{row['rowNumber']}/decision", + json={"decision": decision, "reason": reason})) + + +@st.dialog("Row history", width="large") +def history_dialog(upload_id: str, row_number: int): + entries = call("GET", f"/api/csv/uploads/{upload_id}/rows/{row_number}/history") + if not entries: + st.write("No decisions or edits yet.") + for e in entries: + what = (f"**{e['decision']}**" if e["kind"] == "decision" + else f"**{e['column']}**: `{e['oldValue']}` → `{e['newValue']}`") + st.markdown(f"{e['at'][:19].replace('T', ' ')} · {e['actor']} · {what} \n_{e['reason']}_") + + +# ── Screen ───────────────────────────────────────────────────────────────────── + +try: + uploads = call("GET", "/api/csv/uploads") +except (RuntimeError, requests.RequestException) as exc: + st.error(f"Can't reach the API at {API}: {exc}") + st.stop() +if not uploads: + st.info("No uploads yet.") + st.stop() + +upload = st.selectbox("Upload", uploads, format_func=lambda u: f"#{u['id']} {u['file_name']} ({u['status']})") +uid = upload["id"] +total = upload["total_rows"] or 0 +pages = max(1, -(-total // PAGE)) +page_no = st.number_input(f"Page (of {pages})", 1, pages, 1, key=f"page{uid}") +data = call("GET", f"/api/csv/uploads/{uid}/rows", params={"offset": (page_no - 1) * PAGE, "limit": PAGE}) +columns, rows = data["columns"], data["rows"] + +st.subheader(data["fileName"]) +st.caption(f"{data['total']} rows · {data['excludedRows']} excluded · {data['editedRows']} edited" + + ("" if can_review else " · read-only for your role")) + +grid = pd.DataFrame([r["current"] for r in rows], columns=columns, + index=[r["rowNumber"] for r in rows]) +grid.insert(0, "status", ["Excluded" if r["decision"] == "exclude" else "Invalid" if r["errors"] else "Included" + for r in rows]) + + +def tint(_): + out = pd.DataFrame("", index=grid.index, columns=grid.columns) + for r in rows: + for i in r["editedColumns"]: + out.loc[r["rowNumber"], columns[i]] = "background-color: rgba(59,130,246,.18)" + for e in r["errors"]: + if e["column"] in columns: + out.loc[r["rowNumber"], e["column"]] = "background-color: rgba(239,68,68,.22)" + if r["decision"] == "exclude": + out.loc[r["rowNumber"]] = out.loc[r["rowNumber"]] + ";color:#9ca3af" # st.dataframe ignores opacity + return out + + +picked = st.dataframe(grid.style.apply(tint, axis=None), on_select="rerun", selection_mode="single-row", + key=f"grid{uid}_{page_no}", use_container_width=True, height=560) +sel = picked.selection.rows +if not sel: + st.caption("Select a row to see its errors, original values and history.") + st.stop() + +row = rows[sel[0]] +st.markdown(f"### Row {row['rowNumber']}") +for e in row["errors"]: + st.error(f"**{e['column']}**: {e['reason']}") +if row["decisionReason"]: + st.info(f"Decision: **{row['decision']}**: {row['decisionReason']}") +for i in row["editedColumns"]: + st.caption(f"{columns[i]}: original `{row['original'][i]}`, now `{row['current'][i]}`") + +b = st.columns(4) +if b[0].button("History"): + history_dialog(uid, row["rowNumber"]) +if can_review: + if b[1].button("Edit a cell"): + edit_dialog(uid, row, columns) + flip = "include" if row["decision"] == "exclude" else "exclude" + if b[2].button(flip.title()): + decision_dialog(uid, row, flip) diff --git a/spikes/streamlit_review/requirements.txt b/spikes/streamlit_review/requirements.txt new file mode 100644 index 0000000..e0d878f --- /dev/null +++ b/spikes/streamlit_review/requirements.txt @@ -0,0 +1 @@ +streamlit>=1.40 diff --git a/tests/run_tests.sh b/tests/run_tests.sh index f1d7a48..678e0ae 100644 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -112,7 +112,7 @@ run_suite() { 2>&1 | tee "${outfile}"; then log "Test run for ${env^^} completed." else - error "Test run FAILED for ${env^^} — check output above." + error "Test run FAILED for ${env^^} — psql exited non-zero." FAILED_ENVS+=("${env}") fi @@ -122,6 +122,17 @@ run_suite() { | tail -1 | xargs || echo "unknown") RESULTS["${env}"]="${overall}" + # #181: psql exits 0 even when SQL assertions record FAIL rows. Parse the + # output for failures and mark the environment as failed if any are found. + if grep -qiE 'FAIL|TEST\(S\) FAILED' "${outfile}" && \ + ! grep -q 'ALL TESTS PASSED' "${outfile}"; then + error "Assertion failures detected in ${env^^} output." + # Only add if not already in the list + local already=false + for e in "${FAILED_ENVS[@]:-}"; do [[ "$e" == "$env" ]] && already=true; done + $already || FAILED_ENVS+=("${env}") + fi + rm -f "${outfile}" } diff --git a/tests/test_api_coverage.py b/tests/test_api_coverage.py index a04b006..e76b53b 100644 --- a/tests/test_api_coverage.py +++ b/tests/test_api_coverage.py @@ -389,7 +389,8 @@ def test_response_shape_matches_contract(self): "columns", "rowErrors"): self.assertIn(key, body, f"missing key '{key}'") self.assertEqual(body["duplicateRowsSkipped"], 0) - self.assertEqual(body["types"], []) + # #190: one reviewable type per column (organisations' columns are all text). + self.assertEqual(body["types"], ["text"] * len(body["columns"])) def test_column_mismatch_returns_error(self): csv = "bogus_col,fake_col\nval1,val2\n" @@ -402,7 +403,7 @@ def test_row_level_error_reported(self): csv = self._org_csv(f"{self.tag}_bad", org_type="X" * 100) r = self._upload_te(csv) body = r.json() - self.assertEqual(body["status"], "ok", body) + self.assertEqual(body["status"], "error", body) self.assertGreaterEqual(body["failedRows"], 1) self.assertTrue(body["rowErrors"]) self.assertIn("rowNumber", body["rowErrors"][0]) @@ -424,7 +425,7 @@ def test_enum_constraint_violation_reported_per_row(self): csv = self._org_csv(f"{self.tag}_enum", org_type="INVALID_ENUM") r = self._upload_te(csv) body = r.json() - self.assertEqual(body["status"], "ok", body) + self.assertEqual(body["status"], "error", body) self.assertGreaterEqual(body["failedRows"], 1, "Invalid enum value should fail at row level, not crash the endpoint") self.assertIn("rowNumber", body["rowErrors"][0]) @@ -434,7 +435,7 @@ def test_null_in_not_null_column_reported_per_row(self): csv = "name,org_type,country\n,government,AU\n" r = self._upload_te(csv) body = r.json() - self.assertEqual(body["status"], "ok", body) + self.assertEqual(body["status"], "error", body) self.assertGreaterEqual(body["failedRows"], 1, "NULL in a NOT NULL column should fail at row level") self.assertIn("rowNumber", body["rowErrors"][0]) diff --git a/tests/test_header_mapping.py b/tests/test_header_mapping.py new file mode 100644 index 0000000..2b541a0 --- /dev/null +++ b/tests/test_header_mapping.py @@ -0,0 +1,129 @@ +"""Confirmed header renames reach the database column names (#186).""" + +from __future__ import annotations + +import os +import unittest +import uuid + +import pytest +from fastapi.testclient import TestClient + +from api.main import app +from api.services.csv_parse import apply_header_mapping + +_HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/" + "PGPASSWORD/PGDATABASE). See scripts/start-api.ps1.") + + +@pytest.mark.unit +class ApplyHeaderMapping(unittest.TestCase): + def test_no_mapping_returns_headers_unchanged(self): + self.assertEqual(apply_header_mapping(["a", "b"], None), (["a", "b"], None)) + self.assertEqual(apply_header_mapping(["a", "b"], []), (["a", "b"], None)) + + def test_rename_replaces_only_the_named_header(self): + out, err = apply_header_mapping(["foo", "baz"], [{"original": "foo", "renamed": "bar"}]) + self.assertEqual((out, err), (["bar", "baz"], None)) + + def test_blank_rename_keeps_the_original(self): + out, err = apply_header_mapping( + ["foo", "baz"], + [{"original": "foo", "renamed": ""}, {"original": "baz", "renamed": " "}], + ) + self.assertEqual((out, err), (["foo", "baz"], None)) + + def test_unknown_original_is_rejected(self): + out, err = apply_header_mapping(["foo"], [{"original": "nope", "renamed": "bar"}]) + self.assertEqual(out, ["foo"]) + self.assertIn('"nope"', err) + + def test_original_listed_twice_is_rejected(self): + _, err = apply_header_mapping( + ["foo"], [{"original": "foo", "renamed": "a"}, {"original": "foo", "renamed": "b"}]) + self.assertIn("more than once", err) + + def test_rename_colliding_with_an_untouched_column_is_rejected(self): + out, err = apply_header_mapping(["foo", "bar"], [{"original": "foo", "renamed": "bar"}]) + self.assertEqual(out, ["foo", "bar"]) + self.assertIn('"bar"', err) + + def test_collision_is_judged_after_sanitising(self): + _, err = apply_header_mapping(["foo", "bar"], [{"original": "foo", "renamed": "BAR!"}]) + self.assertIsNotNone(err) + + def test_swapping_two_names_is_allowed(self): + out, err = apply_header_mapping( + ["a", "b"], [{"original": "a", "renamed": "b"}, {"original": "b", "renamed": "a"}]) + self.assertEqual((out, err), (["b", "a"], None)) + + +@pytest.mark.integration +class HeaderMappingOverTheApi(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ctx = TestClient(app) + try: + cls.client = cls.ctx.__enter__() + except Exception as exc: # noqa: BLE001 + raise AssertionError(f"Could not start the API: {type(exc).__name__}: {exc}\n{_HELP}") from None + + @classmethod + def tearDownClass(cls): + cls.ctx.__exit__(None, None, None) + + def setUp(self): + self.tag = f"{os.getpid()}_{uuid.uuid4().hex[:8]}" + + def _upload(self, content: str, **extra) -> dict: + body = {"fileName": f"hdrmap_{self.tag}.csv", "content": content, **extra} + return self.client.post("/api/csv/upload", json=body).json() + + def test_renamed_header_becomes_the_database_column(self): + res = self._upload( + f"foo,baz\n1,{self.tag}\n", + headerMapping=[{"original": "foo", "renamed": "bar"}, {"original": "baz", "renamed": ""}], + ) + self.assertEqual(res["status"], "ok", res) + self.assertEqual(res["columns"], ["bar", "baz"]) + + rows = self.client.get(f"/api/csv/tables/{res['tableName']}/rows").json()["rows"] + self.assertEqual(len(rows), 1) + self.assertIn("bar", rows[0]) + self.assertNotIn("foo", rows[0]) + self.assertEqual(rows[0]["bar"], "1") + + files = self.client.get("/api/csv/files").json() + registered = next(f for f in files if f["table_name"] == res["tableName"]) + self.assertEqual(registered["column_names"], ["bar", "baz"]) + + def test_upload_without_a_mapping_keeps_csv_headers(self): + res = self._upload(f"foo,baz\n1,{self.tag}\n") + self.assertEqual((res["status"], res["columns"]), ("ok", ["foo", "baz"])) + + def test_mapping_for_a_header_not_in_the_file_is_an_error_and_loads_nothing(self): + res = self._upload( + f"foo,baz\n1,{self.tag}\n", + headerMapping=[{"original": "missing", "renamed": "bar"}], + ) + self.assertEqual((res["status"], res["reason"]), ("error", "invalid_header_mapping")) + files = self.client.get("/api/csv/files").json() + self.assertFalse([f for f in files if f["file_name"] == f"hdrmap_{self.tag}.csv"]) + + def test_rename_that_collides_with_another_column_is_an_error(self): + res = self._upload( + f"foo,bar\n1,{self.tag}\n", + headerMapping=[{"original": "foo", "renamed": "bar"}], + ) + self.assertEqual((res["status"], res["reason"]), ("error", "invalid_header_mapping")) + + def test_te_mode_rejects_a_header_mapping_instead_of_ignoring_it(self): + resp = self.client.post("/api/csv/upload", json={ + "fileName": f"hdrmap_te_{self.tag}.csv", + "content": "a,b\n1,2\n", + "mode": "te", + "targetTable": "organisations", + "headerMapping": [{"original": "a", "renamed": "b"}], + }) + self.assertEqual(resp.status_code, 422) + self.assertIn("dynamic", resp.json()["detail"]) diff --git a/tests/test_original_bytes.py b/tests/test_original_bytes.py new file mode 100644 index 0000000..9447ea0 --- /dev/null +++ b/tests/test_original_bytes.py @@ -0,0 +1,101 @@ +"""The preserved original is the uploaded bytes, not a UTF-8 re-encoding of decoded text (#185; BR-24). + +A BOM or an invalid byte changes the retained bytes and the digest if the original is +rebuilt from the decoded text. When the client sends the raw bytes alongside the text, +the stored original and its sha256 must match the uploader's own copy. + +Integration: needs a live PostgreSQL. Fails (never skips) without one. +""" + +from __future__ import annotations + +import base64 +import hashlib +import os +import unittest +import uuid + +import pytest +from fastapi.testclient import TestClient + +from api.main import app + +_HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/" + "PGPASSWORD/PGDATABASE). See scripts/start-api.ps1.") + + +def _browser_decode(raw: bytes) -> str: + """What FileReader.readAsText hands the client for UTF-8 input: BOM dropped, bad bytes -> U+FFFD.""" + text = raw.decode("utf-8", errors="replace") + return text[1:] if text.startswith("") else text + + +@pytest.mark.integration +class OriginalBytesAreKept(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ctx = TestClient(app) + try: + cls.client = cls.ctx.__enter__() + except Exception as exc: # noqa: BLE001 + raise AssertionError(f"Could not start the API: {type(exc).__name__}: {exc}\n{_HELP}") from None + + @classmethod + def tearDownClass(cls): + cls.ctx.__exit__(None, None, None) + + def setUp(self): + self.tag = f"{os.getpid()}_{uuid.uuid4().hex[:8]}" + + def _upload_bytes(self, raw: bytes, **extra) -> dict: + body = { + "fileName": f"bytes_{self.tag}.csv", + "content": _browser_decode(raw), + "contentBase64": base64.b64encode(raw).decode("ascii"), + **extra, + } + return self.client.post("/api/csv/upload", json=body).json() + + def _assert_round_trips(self, raw: bytes, res: dict): + original = self.client.get(f"/api/csv/uploads/{res['uploadId']}/original") + self.assertEqual(original.content, raw) + self.assertEqual(original.headers["x-content-sha256"], hashlib.sha256(raw).hexdigest()) + stored = next(u for u in self.client.get("/api/csv/uploads").json() if u["id"] == res["uploadId"]) + self.assertEqual((stored["sha256"], stored["byte_size"]), (hashlib.sha256(raw).hexdigest(), len(raw))) + + def test_bom_prefixed_csv_round_trips_byte_identical(self): + raw = b"\xef\xbb\xbf" + f"a,b\n1,{self.tag}\n".encode() + res = self._upload_bytes(raw) + self.assertEqual(res["status"], "ok", res) + self._assert_round_trips(raw, res) + self.assertEqual(res["columns"], ["a", "b"], "the BOM must not leak into the first column name") + + def test_csv_with_an_invalid_utf8_byte_round_trips_byte_identical(self): + raw = b"a,b\n1," + self.tag.encode() + b"\x80\n" + res = self._upload_bytes(raw) + self.assertEqual(res["status"], "ok", res) + self._assert_round_trips(raw, res) + + def test_the_decoded_view_still_feeds_the_loader_and_source_rows(self): + raw = b"\xef\xbb\xbf" + f"a,b\n1,{self.tag}\n".encode() + res = self._upload_bytes(raw) + rows = self.client.get(f"/api/csv/tables/{res['tableName']}/rows").json()["rows"] + self.assertEqual([r["b"] for r in rows], [self.tag]) + + def test_a_failed_upload_keeps_the_raw_original_too(self): + raw = b"\xef\xbb\xbf" + f"n,t\nx,{self.tag}\n".encode() + res = self._upload_bytes(raw, types=["int8", "text"]) + self.assertEqual(res["status"], "error", res) + self._assert_round_trips(raw, res) + + def test_without_contentbase64_the_utf8_encoding_of_content_is_kept_as_before(self): + content = f"a,b\n1,{self.tag}\n" + res = self.client.post("/api/csv/upload", json={ + "fileName": f"bytes_plain_{self.tag}.csv", "content": content}).json() + self._assert_round_trips(content.encode(), res) + + def test_invalid_base64_is_rejected_before_anything_loads(self): + resp = self.client.post("/api/csv/upload", json={ + "fileName": f"bytes_bad_{self.tag}.csv", "content": "a\n1\n", "contentBase64": "not base64!!"}) + self.assertEqual(resp.status_code, 422) + self.assertIn("base64", resp.json()["detail"]) diff --git a/tests/test_ragged_rows.py b/tests/test_ragged_rows.py index db5bd4b..28bedaa 100644 --- a/tests/test_ragged_rows.py +++ b/tests/test_ragged_rows.py @@ -56,6 +56,12 @@ def test_dynamic_mode_reports_a_row_wider_than_the_header(self): err = res["rowErrors"][0] self.assertEqual((err["rowNumber"], err["value"]), (2, "LOST")) self.assertIn("header has 2", err["reason"]) + # #192: accounting identity — every row is accounted for + self.assertEqual( + res["totalRows"], + res["insertedRows"] + res.get("duplicateRowsSkipped", 0) + res["failedRows"], + f"Accounting identity violated: {res}", + ) def test_dynamic_mode_accepts_a_trailing_empty_cell(self): res = self._upload(f"a,b\n1,{self.tag},\n") @@ -68,3 +74,24 @@ def test_te_mode_reports_a_row_wider_than_the_header(self): self.assertEqual(res["status"], "ok") self.assertEqual((res["insertedRows"], res["failedRows"]), (1, 1)) self.assertEqual(res["rowErrors"][0]["value"], "LOST") + # #192: accounting identity + self.assertEqual( + res["totalRows"], + res["insertedRows"] + res.get("duplicateRowsSkipped", 0) + res["failedRows"], + f"Accounting identity violated: {res}", + ) + + def test_multiple_ragged_rows_each_get_a_reason(self): + """Every ragged row gets its own entry in rowErrors with a reason.""" + content = (f"a,b\n" + f"1,{self.tag}\n" + f"2,{self.tag},X1,X2\n" + f"3,{self.tag},X3\n") + res = self._upload(content) + self.assertEqual(res["status"], "ok") + self.assertEqual(res["failedRows"], 2) + self.assertEqual(res["insertedRows"], 1) + for err in res["rowErrors"]: + self.assertIn("rowNumber", err, f"Missing rowNumber: {err}") + self.assertIn("reason", err, f"Missing reason: {err}") + self.assertTrue(err["reason"], f"Empty reason: {err}") diff --git a/tests/test_registry_race.py b/tests/test_registry_race.py new file mode 100644 index 0000000..a40b8e2 --- /dev/null +++ b/tests/test_registry_race.py @@ -0,0 +1,95 @@ +"""Concurrent uploads of the same file must not create duplicate registry rows (#191). + +The UNIQUE constraints on csv_files(file_hash) and csv_files(file_name) +settle the race at the database level. This test verifies that two +simultaneous uploads of the same content yield one success and one +graceful rejection — not a duplicate row or partial state. +""" + +from __future__ import annotations + +import os +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed + +import pytest +from fastapi.testclient import TestClient + +from api.main import app + +_HELP = ("Start PostgreSQL and ensure api/ can reach it (PGHOST/PGPORT/PGUSER/" + "PGPASSWORD/PGDATABASE). See scripts/start-api.ps1.") + + +@pytest.mark.integration +class RegistryRace(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ctx = TestClient(app) + try: + cls.client = cls.ctx.__enter__() + except Exception as exc: # noqa: BLE001 + raise AssertionError( + f"Could not start the API: {type(exc).__name__}: {exc}\n{_HELP}" + ) from None + + @classmethod + def tearDownClass(cls): + cls.ctx.__exit__(None, None, None) + + def setUp(self): + self.tag = f"{os.getpid()}_{uuid.uuid4().hex[:8]}" + + def _upload(self, file_name: str, content: str) -> dict: + body = {"fileName": file_name, "content": content} + return self.client.post("/api/csv/upload", json=body).json() + + def test_concurrent_identical_uploads_one_wins(self): + """Two threads upload the same content simultaneously. + + One should succeed; the other should fail gracefully (duplicate + hash detected). The registry must contain exactly one row. + """ + fname = f"race_same_{self.tag}.csv" + content = f"a,b\n1,{self.tag}\n" + + results: list[dict] = [] + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(self._upload, fname, content) for _ in range(2)] + for f in as_completed(futures): + results.append(f.result()) + + statuses = [r.get("status") for r in results] + ok_count = statuses.count("ok") + self.assertGreaterEqual(ok_count, 1, f"At least one upload must succeed: {results}") + self.assertLessEqual(ok_count + statuses.count("error"), 2) + + files_resp = self.client.get("/api/csv/files").json() + matches = [f for f in files_resp if f.get("file_name") == fname] + self.assertEqual(len(matches), 1, f"Registry must have exactly one row: {matches}") + + def test_concurrent_same_name_different_content(self): + """Two threads upload different content under the same filename. + + The UNIQUE(file_name) constraint means only one can register. The + other must fail or be treated as a reupload (depending on the + code path), but not leave a second registry row. + """ + fname = f"race_name_{self.tag}.csv" + content_a = f"a,b\nA,{self.tag}\n" + content_b = f"x,y\nB,{self.tag}\n" + + results: list[dict] = [] + with ThreadPoolExecutor(max_workers=2) as pool: + fa = pool.submit(self._upload, fname, content_a) + fb = pool.submit(self._upload, fname, content_b) + for f in as_completed([fa, fb]): + results.append(f.result()) + + files_resp = self.client.get("/api/csv/files").json() + matches = [f for f in files_resp if f.get("file_name") == fname] + self.assertLessEqual( + len(matches), 1, + f"At most one registry row per filename: {matches}", + ) diff --git a/tests/test_review.py b/tests/test_review.py index 92a159a..1b40730 100644 --- a/tests/test_review.py +++ b/tests/test_review.py @@ -109,6 +109,30 @@ def test_refusals(self): self.assertEqual(res.status_code, code) self.assertEqual(self.client.get(f"/api/csv/uploads/{uid}/rows/2/history").json(), []) + def test_te_review_shows_the_database_reason_a_row_was_rejected(self): + # #190: T&E uploads stored no types, so review said errors: [] for a row the CHECK rejected. + good, bad = f"TE_REV_{self.tag}_ok", f"TE_REV_{self.tag}_bad" + res = self.client.post("/api/csv/upload", json={ + "fileName": f"te_rev_{self.tag}.csv", "mode": "te", "targetTable": "organisations", + "content": f"name,org_type,country\n{good},government,AU\n{bad},not_a_type,AU\n"}).json() + try: + self.assertEqual(res["failedRows"], 1, res) + page = self._rows(res["uploadId"]) + self.assertEqual(len(page["types"]), 3) + ok_row, bad_row = page["rows"] + self.assertEqual(ok_row["errors"], []) + [err] = bad_row["errors"] + self.assertEqual(err["column"], "(database)") + self.assertIn("org_type", err["reason"]) + # Once a reviewer edits the row, the upload-time reason no longer describes it. + self.assertEqual(self._edit(res["uploadId"], 2, "org_type", "prime").status_code, 200) + self.assertEqual(self._rows(res["uploadId"])["rows"][1]["errors"], []) + finally: + with Conn() as conn, conn.cursor() as cur: + cur.execute(sql.SQL("DELETE FROM {}.organisations WHERE name = ANY(%s)").format( + sql.Identifier(settings.TE_SCHEMA)), ([good, bad],)) + conn.commit() + def test_changes_are_audited_and_insert_only(self): uid = self._three_rows() self._decide(uid, 3, "exclude")