From 5a3be9d0091ae644cfacea6d1a025535a16ac8d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 14:11:52 +0000 Subject: [PATCH] ci: accessibility job, Playwright + axe-core WCAG 2.1 AA scans (#168) New Quality Gate job `accessibility`: tests/accessibility/ scans the frontend's main pages in Chromium against the Vite dev server, and only there. The API is stubbed, so the job needs no database and a backend-only change cannot alter what is scanned. Nine scenarios (sign-in, upload page empty / with history / mapping dialog / toast, audit log and its menu, review page) each wait for content that proves the page rendered before scanning. The frontend has violations today, so the scans judge against tests/accessibility/known_violations.json: a new violation, or more elements than recorded, fails; a recorded one that is fixed fails until its entry is deleted. Emptying the list makes the job strict. The integration job, make test-all and run_qa.ps1 exclude the marker. requirements-a11y.txt keeps the browser driver out of the other jobs. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01VBrxqChRJtxdvSpFhiUWUy --- .github/workflows/quality-gate.yml | 66 ++++++- Makefile | 5 +- docs/TEST_CASES.md | 102 ++++++++++- requirements-a11y.txt | 10 + scripts/run_qa.ps1 | 2 +- scripts/select_tests.py | 1 + tests/README.md | 2 +- tests/accessibility/README.md | 83 +++++++++ tests/accessibility/axe_baseline.py | 96 ++++++++++ tests/accessibility/conftest.py | 212 ++++++++++++++++++++++ tests/accessibility/known_violations.json | 65 +++++++ tests/accessibility/stub_api.py | 125 +++++++++++++ tests/accessibility/test_accessibility.py | 111 +++++++++++ tests/unit/test_a11y_baseline.py | 123 +++++++++++++ tests/unit/test_ci_quality_gate.py | 30 +++ 15 files changed, 1026 insertions(+), 7 deletions(-) create mode 100644 requirements-a11y.txt create mode 100644 tests/accessibility/README.md create mode 100644 tests/accessibility/axe_baseline.py create mode 100644 tests/accessibility/conftest.py create mode 100644 tests/accessibility/known_violations.json create mode 100644 tests/accessibility/stub_api.py create mode 100644 tests/accessibility/test_accessibility.py create mode 100644 tests/unit/test_a11y_baseline.py diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 4bfab3c..dff89e2 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -164,17 +164,20 @@ jobs: # the build on any skip. # The API reads PGDATABASE (api/config.py). The schema lives in # te_mgmt_dev, not the job-level default "postgres" (issue #45). + # Accessibility scans need a browser and the frontend, not this database; + # they run in the `accessibility` job below (#168) and are reported here as + # NOT RUN, not as skipped. - name: Full test suite — final result with skip accounting env: PGDATABASE: te_mgmt_dev - run: python3 scripts/test_report.py --strict + run: python3 scripts/test_report.py --markers "not accessibility" --strict # #194: measure api/services coverage with per-loader floors. - name: api/services coverage gate env: PGDATABASE: te_mgmt_dev run: | - python -m pytest tests/ \ + python -m pytest tests/ -m "not accessibility" \ --cov=api/services \ --cov-report=json:coverage-services.json \ --cov-report=term-missing \ @@ -189,3 +192,62 @@ jobs: path: evals/reports/ if-no-files-found: ignore retention-days: 30 + + # #168: WCAG 2.1 A and AA scans of the frontend (csv-table-hub-main) with + # Playwright + axe-core in Chromium, against the Vite dev server. The API is + # stubbed, so this job needs no database and a backend-only change cannot + # alter what it scans. Known violations are recorded in + # tests/accessibility/known_violations.json: the job fails on a NEW violation + # and when a recorded one is fixed (so the list only shrinks). Not yet a + # required check; make it one after a few green runs (see the issue). + accessibility: + name: accessibility (Playwright + axe-core, WCAG 2.1 AA) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.11' + cache: 'pip' + cache-dependency-path: | + requirements-dev.txt + requirements-a11y.txt + api/requirements.txt + + - name: Setup Node + uses: actions/setup-node@v6 + with: + node-version: '22' + cache: 'npm' + cache-dependency-path: csv-table-hub-main/package-lock.json + + - name: Install dev dependencies + # api/requirements.txt is needed too: tests/conftest.py imports the API + # app, so pytest collection fails without FastAPI. + run: pip install -r requirements-dev.txt -r api/requirements.txt -r requirements-a11y.txt + + - name: Install frontend dependencies + working-directory: csv-table-hub-main + run: npm ci + + - name: Install Chromium for Playwright + run: python -m playwright install --with-deps chromium + + # Prints the same final block as the other jobs. --strict fails on any skip. + # The scans start `npm run dev` themselves (tests/accessibility/conftest.py). + - name: Accessibility scans — final result with skip accounting + env: + A11Y_REPORT_DIR: accessibility-report + run: python3 scripts/test_report.py --markers accessibility --strict + + - name: Upload accessibility report + if: always() + uses: actions/upload-artifact@v7 + with: + name: accessibility-report + path: accessibility-report/ + if-no-files-found: ignore + retention-days: 30 diff --git a/Makefile b/Makefile index ca4075b..09892c7 100644 --- a/Makefile +++ b/Makefile @@ -27,10 +27,11 @@ test-evals: test-e2e: pytest -m "e2e or integration or parity" tests/ -v -# All: every test tier including evals. +# All: every test tier including evals. Accessibility scans need a browser and the +# frontend, so they run on their own: `make test-accessibility` (tests/accessibility/README.md). # Mirrors: bun run test:evals:all test-all: - pytest tests/ --cov=build/csv --cov=evals --cov-report=term-missing + pytest tests/ -m "not accessibility" --cov=build/csv --cov=evals --cov-report=term-missing python3 evals/runner.py --tiers p,i,s,e --verbose # ── One category at a time (#167) ───────────────────────────────────────────── diff --git a/docs/TEST_CASES.md b/docs/TEST_CASES.md index db391bd..9470571 100644 --- a/docs/TEST_CASES.md +++ b/docs/TEST_CASES.md @@ -1,10 +1,34 @@ # Test Catalogue > Auto-generated by `scripts/test_catalogue.py` — do not edit by hand. -> 726 tests across 37 files. +> 772 tests across 39 files. To regenerate: `python3 scripts/test_catalogue.py` +## `tests/accessibility/test_accessibility.py` + +| Test | Markers | Objective | +|---|---|---| +| `test_sign_in_page` | — | The sign-in page, before anyone is signed in. | + +| `test_sign_in_page_after_a_failed_attempt` | — | The sign-in page showing the server's refusal. | + +| `test_upload_page_with_nothing_uploaded_yet` | — | The upload page for a new tenant: no migrated files and no import history. | + +| `test_upload_page_with_files_and_import_history` | — | The upload page with a migrated file and one import of each kind (done, failed, duplicate). | + +| `test_upload_page_column_mapping_dialog` | — | The dialog that opens when a CSV is chosen: column names and the type of each. | + +| `test_upload_page_rejection_toast` | — | The toast shown when a file that is not a CSV is chosen. | + +| `test_audit_log_with_history` | — | The audit log listing one import of each kind. | + +| `test_audit_log_report_menu` | — | The audit log with its report-download menu open. | + +| `test_review_page` | — | The review screen for one stored upload, with a row that failed validation. | + +| `test_signed_out_visitors_are_sent_to_sign_in` | — | A page that needs a session sends a visitor with none to the sign-in page. | + ## `tests/system/test_accounts.py` | Test | Markers | Objective | @@ -1089,6 +1113,66 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_dev_row_counts_stable_after_second_deploy` | — | — | +## `tests/unit/test_a11y_baseline.py` + +| Test | Markers | Objective | +|---|---|---| +| `Compare.test_nothing_found_and_nothing_recorded_is_ok` | — | — | + +| `Compare.test_a_violation_not_in_the_baseline_is_new_and_fails` | — | — | + +| `Compare.test_the_baseline_is_per_scenario` | — | — | + +| `Compare.test_exactly_as_recorded_is_known_and_passes` | — | — | + +| `Compare.test_more_elements_than_recorded_is_a_regression` | — | — | + +| `Compare.test_fewer_elements_is_a_notice_not_a_failure` | — | — | + +| `Compare.test_a_recorded_rule_that_no_longer_fails_must_be_removed` | — | — | + +| `Compare.test_new_and_fixed_are_reported_together` | — | — | + +| `Describe.test_lists_a_few_elements_and_counts_the_rest` | — | — | + +| `LoadBaseline.test_rejects_a_rule_listed_twice_for_a_scenario` | — | — | + +| `LoadBaseline.test_rejects_an_entry_without_a_positive_count` | — | — | + +| `LoadBaseline.test_rejects_an_entry_without_a_note` | — | — | + +| `RecordedBaseline.test_is_valid_and_names_real_scenarios` | — | — | + +| `RecordedBaseline.test_scans_wcag_21_a_and_aa` | — | — | + +| `test_nothing_found_and_nothing_recorded_is_ok` | — | — | + +| `test_a_violation_not_in_the_baseline_is_new_and_fails` | — | — | + +| `test_the_baseline_is_per_scenario` | — | — | + +| `test_exactly_as_recorded_is_known_and_passes` | — | — | + +| `test_more_elements_than_recorded_is_a_regression` | — | — | + +| `test_fewer_elements_is_a_notice_not_a_failure` | — | — | + +| `test_a_recorded_rule_that_no_longer_fails_must_be_removed` | — | — | + +| `test_new_and_fixed_are_reported_together` | — | — | + +| `test_lists_a_few_elements_and_counts_the_rest` | — | — | + +| `test_rejects_a_rule_listed_twice_for_a_scenario` | — | — | + +| `test_rejects_an_entry_without_a_positive_count` | — | — | + +| `test_rejects_an_entry_without_a_note` | — | — | + +| `test_is_valid_and_names_real_scenarios` | — | — | + +| `test_scans_wcag_21_a_and_aa` | — | — | + ## `tests/unit/test_api_hardening.py` | Test | Markers | Objective | @@ -1261,6 +1345,14 @@ To regenerate: `python3 scripts/test_catalogue.py` | `TestAllJobsHaveConsistentStructure.test_all_jobs_install_dev_deps` | — | — | +| `TestAccessibilityJob.test_has_accessibility_job_that_runs_only_the_accessibility_marker` | — | — | + +| `TestAccessibilityJob.test_accessibility_job_installs_a_browser_and_the_frontend` | — | — | + +| `TestAccessibilityJob.test_integration_job_excludes_accessibility_everywhere_it_runs_the_whole_tree` | — | — | + +| `TestAccessibilityJob.test_free_tier_markers_cannot_select_accessibility_tests` | — | — | + | `TestGapAnalysisG2Accepted.test_g2_row_exists` | — | — | | `TestGapAnalysisG2Accepted.test_g2_is_accepted_or_deferred` | — | — | @@ -1285,6 +1377,14 @@ To regenerate: `python3 scripts/test_catalogue.py` | `test_all_jobs_install_dev_deps` | — | — | +| `test_has_accessibility_job_that_runs_only_the_accessibility_marker` | — | — | + +| `test_accessibility_job_installs_a_browser_and_the_frontend` | — | — | + +| `test_integration_job_excludes_accessibility_everywhere_it_runs_the_whole_tree` | — | — | + +| `test_free_tier_markers_cannot_select_accessibility_tests` | — | — | + | `test_g2_row_exists` | — | — | | `test_g2_is_accepted_or_deferred` | — | — | diff --git a/requirements-a11y.txt b/requirements-a11y.txt new file mode 100644 index 0000000..63406b9 --- /dev/null +++ b/requirements-a11y.txt @@ -0,0 +1,10 @@ +# Accessibility scans (tests/accessibility/, the `accessibility` job in +# .github/workflows/quality-gate.yml). Kept apart from requirements-dev.txt so the +# other jobs, and developers who never scan, do not install a browser driver. +# +# pip install -r requirements-a11y.txt +# python -m playwright install chromium +# (cd csv-table-hub-main && npm ci) +# python -m pytest tests/accessibility -v +playwright>=1.64.1 +axe-playwright-python>=0.1.8 diff --git a/scripts/run_qa.ps1 b/scripts/run_qa.ps1 index c1cf60d..28b1379 100644 --- a/scripts/run_qa.ps1 +++ b/scripts/run_qa.ps1 @@ -34,7 +34,7 @@ switch ($Target) { pytest -m "e2e or integration or parity" tests/ -v } "test-all" { - pytest tests/ --cov=build/csv --cov=evals --cov-report=term-missing + pytest tests/ -m "not accessibility" --cov=build/csv --cov=evals --cov-report=term-missing python evals/runner.py --tiers p,i,s --verbose } "lint" { diff --git a/scripts/select_tests.py b/scripts/select_tests.py index 1914e46..1897218 100644 --- a/scripts/select_tests.py +++ b/scripts/select_tests.py @@ -36,6 +36,7 @@ ("tests/system/", "integration", "system test changed → integration"), ("tests/system_integration/", "e2e or integration or parity", "system-integration test changed → e2e + integration + parity"), ("tests/accessibility/", "accessibility", "accessibility test changed → accessibility"), + ("csv-table-hub-main/src/", "accessibility", "frontend changed → accessibility scans (CI job only)"), ("scripts/", "security", "scripts changed → security scan"), ("build/adapters/", "security", "adapter changed → security scan"), ("build/csv/loader_", "e2e", "loader changed → e2e"), diff --git a/tests/README.md b/tests/README.md index ac120c2..2eec9d7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -10,7 +10,7 @@ automatically. | `system/` | The API and a live PostgreSQL | `system` (automatic) | `make test-system` | | `system_integration/` | The whole pipeline: shell loaders, every deployed environment, cross-environment parity | `sit` (automatic) | `make test-sit` | | `performance/` | Load and timing tests (none yet) | `performance` (automatic) | `make test-performance` | -| `accessibility/` | A real browser against the running app, WCAG scans (#168) | `accessibility` (automatic) | `make test-accessibility` | +| `accessibility/` | Chromium, the frontend's dev server and Node; the API is stubbed. WCAG 2.1 AA scans, own CI job ([README](accessibility/README.md), #168) | `accessibility` (automatic) | `make test-accessibility` | | `uat/` | User-acceptance scenarios (none yet) | `uat` (automatic) | `make test-uat` | Which folder does a new test go in? Ask what is the least it needs: nothing → diff --git a/tests/accessibility/README.md b/tests/accessibility/README.md new file mode 100644 index 0000000..af561b0 --- /dev/null +++ b/tests/accessibility/README.md @@ -0,0 +1,83 @@ +# Accessibility scans (#168) + +WCAG 2.1 level A and AA checks of the frontend (`csv-table-hub-main`), run with +[axe-core](https://github.com/dequelabs/axe-core) in Chromium through Playwright, against the +Vite dev server. They run in their own CI job, `accessibility` in +`.github/workflows/quality-gate.yml`, and nowhere else: the integration job runs with +`-m "not accessibility"`. + +## Run them + +```bash +pip install -r requirements-a11y.txt +python -m playwright install chromium +(cd csv-table-hub-main && npm ci) + +python -m pytest tests/accessibility -v # or: make test-accessibility +``` + +The scans start `npm run dev` themselves on a free port and stop it afterwards. Missing +Playwright, Chromium or `node_modules` fails the run with the command to fix it; it is never a +skip. + +| Variable | Effect | +|---|---| +| `A11Y_BASE_URL` | Scan a frontend that is already running instead of starting one | +| `A11Y_CHROMIUM_PATH` | Use this Chromium instead of the one `playwright install` fetched | +| `A11Y_REPORT_DIR` | Write one JSON file per scenario with every violation axe-core found (CI uploads these as the `accessibility-report` artifact) | + +## What is scanned + +The API is stubbed in `stub_api.py`: no database, no API process. Every `/api/**` request gets a +fixed answer and a signed-in session is planted in the browser. A backend-only change therefore +cannot alter what is scanned, and the job needs no PostgreSQL service. + +| Scenario | State | +|---|---| +| `sign_in`, `sign_in_failed` | the sign-in page, and after the server refuses a login | +| `upload_page_empty` | a new tenant: no files, no import history | +| `upload_page_with_history` | a migrated file and one import of each kind (done, failed, duplicate) | +| `upload_mapping_dialog` | the dialog that opens when a CSV is chosen | +| `upload_page_toast` | the toast shown when a non-CSV file is chosen | +| `audit_log_with_history`, `audit_log_report_menu` | the audit log, and with its report menu open | +| `review_page` | the review screen for one stored upload | + +Each test waits for content that proves the page rendered before it scans. A page that +redirected to sign-in or showed an error would otherwise scan clean and pass for the wrong +reason. `test_signed_out_visitors_are_sent_to_sign_in` guards the redirect itself. + +Not covered: dark mode, mobile widths, keyboard-only flows (axe-core checks markup, not whether +a task can be completed), and the Streamlit spike (#154). + +## Known violations, and why the job passes today + +The frontend has accessibility problems, and fixing them is separate work (out of scope for +#168). A scan that fails on any violation would keep the job red until then, and a job that is +always red gets ignored. So `known_violations.json` records what is wrong now, per scenario and +rule with an element count, and each scan fails on a **change**: + +* a rule that fails where it is not recorded, or on more elements than recorded: **fails** + (a regression); +* a recorded rule that no longer fails: **fails** until its entry is deleted, so the list can only + get shorter. The message names the entry; +* fewer elements than recorded: reported only. Lower the number when convenient. + +When a frontend change fixes a violation, delete its entry in the same pull request. To see the +whole picture, including what is recorded, set `A11Y_REPORT_DIR` and read the JSON, or run with +`-s` to print the known violations. + +**To make the job strict instead** (red until the frontend is fixed), empty the `violations` +list. That is the only change needed. + +The recorded violations are the ones the closed PR #41 bundled with its scan: unnamed selects and +icon-only buttons, unnamed upload progress bars, and low-contrast text (including the error +toast). + +## A frontend bug these scans work around + +In development, React StrictMode runs the effects in +`csv-table-hub-main/src/hooks/use-local-storage.ts` twice: the first pass writes the hook's empty +default over the stored import history, and the second reads that back, so saved history is never +shown after a reload. Production builds do not double-run effects. `stub_api.seed_script` +drops writes of `[]` to that key so the audit page can be scanned with rows in it; without that, the +page was scanned empty and passed without checking a single row. Remove the guard when the hook is fixed. diff --git a/tests/accessibility/axe_baseline.py b/tests/accessibility/axe_baseline.py new file mode 100644 index 0000000..1bfadbc --- /dev/null +++ b/tests/accessibility/axe_baseline.py @@ -0,0 +1,96 @@ +"""Compare axe-core results with the recorded baseline of known violations (#168). + +The frontend has accessibility problems today, and fixing them is a separate piece +of work (see tests/accessibility/README.md). A scan that simply fails on any violation +would keep the accessibility job red until then, so nobody would look at it. Instead the +violations that exist now are written down in ``known_violations.json`` and the scan +fails on a *change*: + +* a rule that fails on a page where it is not in the baseline, or on more elements than + the baseline records, is NEW -> the test fails (a regression); +* a baseline entry whose rule no longer fails at all is FIXED -> the test fails until the + entry is deleted, so the baseline can only get shorter; +* fewer elements than recorded is only reported (lower the number when convenient). + +Pure functions, no browser and no Playwright import: tests/unit/test_a11y_baseline.py +covers them in the free tier. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path + +BASELINE_PATH = Path(__file__).with_name("known_violations.json") + +# WCAG 2.1 level A and AA: the rule tags axe-core is asked to run. +WCAG_21_AA_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"] + +Baseline = dict[tuple[str, str], dict] + + +@dataclass +class Outcome: + new: list[str] = field(default_factory=list) # regressions: fail + fixed: list[str] = field(default_factory=list) # stale baseline entries: fail + shrunk: list[str] = field(default_factory=list) # fewer elements than recorded: notice + known: list[str] = field(default_factory=list) # as recorded: notice + + @property + def ok(self) -> bool: + return not self.new and not self.fixed + + +def load_baseline(path: Path = BASELINE_PATH) -> Baseline: + """The baseline keyed by (scenario, rule id).""" + entries = json.loads(path.read_text(encoding="utf-8"))["violations"] + baseline: Baseline = {} + for entry in entries: + key = (entry["scenario"], entry["rule"]) + if key in baseline: + raise ValueError(f"{path.name}: {key[1]} is listed twice for scenario {key[0]!r}") + if not isinstance(entry.get("nodes"), int) or entry["nodes"] < 1: + raise ValueError(f"{path.name}: {key} needs a positive integer 'nodes'") + if not entry.get("note"): + raise ValueError(f"{path.name}: {key} needs a 'note' saying what is wrong") + baseline[key] = entry + return baseline + + +def describe(violation: dict, limit: int = 3) -> str: + """One violation as a few readable lines: rule, impact, count, and where.""" + nodes = violation["nodes"] + lines = [f"{violation['id']} ({violation.get('impact') or 'n/a'}) on {len(nodes)} element(s): " + f"{violation['help']}"] + for node in nodes[:limit]: + target = " ".join(str(t) for t in node["target"]) + html = " ".join(node.get("html", "").split()) + lines.append(f" {target}\n {html[:140]}") + if len(nodes) > limit: + lines.append(f" ... and {len(nodes) - limit} more") + lines.append(f" {violation.get('helpUrl', '')}") + return "\n".join(lines) + + +def compare(scenario: str, violations: list[dict], baseline: Baseline) -> Outcome: + """Classify the violations found in one scenario against the baseline.""" + out = Outcome() + found = {v["id"]: v for v in violations} + + for rule, v in sorted(found.items()): + count = len(v["nodes"]) + entry = baseline.get((scenario, rule)) + if entry is None: + out.new.append(f"NEW {describe(v)}") + elif count > entry["nodes"]: + out.new.append(f"MORE ELEMENTS than the baseline's {entry['nodes']}: {describe(v)}") + elif count < entry["nodes"]: + out.shrunk.append(f"{rule}: {count} element(s), baseline records {entry['nodes']}") + else: + out.known.append(f"{rule}: {count} element(s), known ({entry['note']})") + + for (scen, rule), entry in sorted(baseline.items()): + if scen == scenario and rule not in found: + out.fixed.append(f"{rule} no longer fails here: delete its entry from known_violations.json " + f"(it recorded: {entry['note']})") + return out diff --git a/tests/accessibility/conftest.py b/tests/accessibility/conftest.py new file mode 100644 index 0000000..1dfa27c --- /dev/null +++ b/tests/accessibility/conftest.py @@ -0,0 +1,212 @@ +"""Playwright fixtures for the accessibility scans (#168). + +What a scan needs: the frontend (csv-table-hub-main) served by the Vite dev server, a +Chromium, and axe-core. The API is stubbed (tests/accessibility/stub_api.py), so there is +no database and no API process. + +Everything heavy is imported inside a fixture, never at module level. ``tests/`` is +collected as a whole by other jobs and by developers who never scan; collecting this +folder must not need Playwright. A missing prerequisite FAILS the test with what to run +(this project's policy: unmet prerequisites are failures, never skips). + +Environment +----------- +A11Y_BASE_URL scan an already running frontend instead of starting ``npm run dev`` +A11Y_CHROMIUM_PATH use this Chromium instead of the one ``playwright install`` fetched +A11Y_REPORT_DIR write one JSON file per scenario with every violation axe found +""" +from __future__ import annotations + +import json +import os +import shutil +import signal +import socket +import subprocess +import tempfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +from tests.accessibility import axe_baseline, stub_api + +ROOT = Path(__file__).resolve().parents[2] +FRONTEND = ROOT / "csv-table-hub-main" +STARTUP_TIMEOUT_S = 180 # first start also pre-bundles dependencies + + +def _fail(message: str): + raise AssertionError(message) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _wait_until_serving(url: str, proc: subprocess.Popen | None, log: Path) -> None: + deadline = time.monotonic() + STARTUP_TIMEOUT_S + while time.monotonic() < deadline: + if proc is not None and proc.poll() is not None: + _fail(f"The frontend dev server exited with code {proc.returncode} before serving.\n" + f"{log.read_text(errors='replace')[-2000:]}") + try: + with urllib.request.urlopen(f"{url}/auth", timeout=5) as res: + if res.status == 200: + return + except (urllib.error.URLError, OSError): + pass + time.sleep(1) + _fail(f"The frontend did not serve {url}/auth within {STARTUP_TIMEOUT_S}s.\n" + f"{log.read_text(errors='replace')[-2000:] if log.exists() else ''}") + + +@pytest.fixture(scope="session") +def app_url(): + """Base URL of the frontend: A11Y_BASE_URL, or a Vite dev server started for the session.""" + given = os.environ.get("A11Y_BASE_URL") + if given: + url = given.rstrip("/") + _wait_until_serving(url, None, Path(os.devnull)) + yield url + return + + npm = shutil.which("npm") + if not npm: + _fail("npm was not found. Install Node.js 22 to serve the frontend for the scans.") + if not (FRONTEND / "node_modules").is_dir(): + _fail(f"The frontend's dependencies are not installed. Run: (cd {FRONTEND.name} && npm ci)") + + port = _free_port() + log = Path(tempfile.mkdtemp(prefix="a11y-vite-")) / "vite.log" + with log.open("wb") as out: + proc = subprocess.Popen( + [npm, "run", "dev", "--", "--host", "127.0.0.1", "--port", str(port), "--strictPort"], + cwd=FRONTEND, stdout=out, stderr=subprocess.STDOUT, + # An empty API base makes the app call /api on its own origin, where the + # stub answers. A new session lets teardown stop npm, vite and their children. + env={**os.environ, "VITE_API_BASE": "", "BROWSER": "none"}, + start_new_session=(os.name == "posix"), + ) + url = f"http://127.0.0.1:{port}" + try: + _wait_until_serving(url, proc, log) + yield url + finally: + if proc.poll() is None: + try: + if os.name == "posix": + os.killpg(proc.pid, signal.SIGTERM) + else: + proc.terminate() + proc.wait(timeout=15) + except (ProcessLookupError, subprocess.TimeoutExpired): + proc.kill() + + +@pytest.fixture(scope="session") +def a11y_browser(app_url): + """One Chromium for the session, warmed up so Vite's first-load dependency + pre-bundling (which reloads the page once) cannot land in the middle of a scan.""" + try: + from playwright.sync_api import sync_playwright + except ImportError: + _fail("Playwright is not installed. Run: pip install -r requirements-a11y.txt " + "&& python -m playwright install chromium") + pw = sync_playwright().start() + try: + launch = {} + if os.environ.get("A11Y_CHROMIUM_PATH"): + launch["executable_path"] = os.environ["A11Y_CHROMIUM_PATH"] + try: + browser = pw.chromium.launch(**launch) + except Exception as exc: # noqa: BLE001 + _fail(f"Chromium could not start: {exc}\nRun: python -m playwright install chromium") + warm = browser.new_context() + warm.add_init_script(stub_api.seed_script(stub_api.SESSION, stub_api.JOBS)) + warm.route("**/api/**", stub_api.handler(stub_api.default_responses(), [])) + page = warm.new_page() + for path in ("/auth", "/", "/audit", "/review/101"): + try: + page.goto(app_url + path, wait_until="networkidle", timeout=120_000) + except Exception: # noqa: BLE001 the scenarios report a page that really fails + pass + warm.close() + yield browser + browser.close() + finally: + pw.stop() + + +@pytest.fixture +def open_page(a11y_browser, app_url, request): + """``open_page(path, ...)`` -> a Page on the frontend with the stubbed API. + + signed_in plant a session (the pages under / need one) + jobs import history to show (see stub_api.seed_script); None for none + responses extra or replacement API responses, route -> body or callable + """ + contexts = [] + unstubbed: list[str] = [] + + def _open(path: str, *, signed_in: bool = True, jobs: list[dict] | None = None, + responses: stub_api.Responses | None = None): + context = a11y_browser.new_context( + viewport={"width": 1280, "height": 900}, locale="en-AU", timezone_id="UTC", + color_scheme="light", + # Animations change what is on screen mid-scan (and what contrast is measured on). + reduced_motion="reduce", + ) + contexts.append(context) + script = stub_api.seed_script(stub_api.SESSION if signed_in else None, jobs) + if script: + context.add_init_script(script) + context.route("**/api/**", stub_api.handler({**stub_api.default_responses(), **(responses or {})}, + unstubbed)) + page = context.new_page() + page.goto(app_url + path, wait_until="networkidle", timeout=60_000) + return page + + yield _open + for context in contexts: + context.close() + # A request the stub did not know is a page asking for something these scans never + # looked at; say so rather than scanning a page that may be half empty. + assert not unstubbed, f"The page called API routes the accessibility stub has no answer for: {unstubbed}" + + +@pytest.fixture +def scan(): + """``scan(page, scenario)``: run axe-core (WCAG 2.1 A and AA) and judge the result + against known_violations.json. Fails on a new violation or a fixed-but-still-listed one.""" + try: + from axe_playwright_python.sync_playwright import Axe + except ImportError: + _fail("axe-playwright-python is not installed. Run: pip install -r requirements-a11y.txt") + baseline = axe_baseline.load_baseline() + + def _scan(page, scenario: str) -> axe_baseline.Outcome: + page.wait_for_load_state("networkidle") + result = Axe().run(page, options={"runOnly": {"type": "tag", "values": axe_baseline.WCAG_21_AA_TAGS}}) + violations = result.response["violations"] + outcome = axe_baseline.compare(scenario, violations, baseline) + + report_dir = os.environ.get("A11Y_REPORT_DIR") + if report_dir: + Path(report_dir).mkdir(parents=True, exist_ok=True) + (Path(report_dir) / f"{scenario}.json").write_text( + json.dumps({"scenario": scenario, "url": page.url, "violations": violations, + "known": outcome.known, "shrunk": outcome.shrunk}, indent=2), + encoding="utf-8") + + for line in outcome.known + outcome.shrunk: + print(f"[a11y {scenario}] {line}") + problems = outcome.new + outcome.fixed + assert not problems, f"[{scenario}] accessibility baseline changed:\n" + "\n".join(problems) + return outcome + + return _scan diff --git a/tests/accessibility/known_violations.json b/tests/accessibility/known_violations.json new file mode 100644 index 0000000..a282978 --- /dev/null +++ b/tests/accessibility/known_violations.json @@ -0,0 +1,65 @@ +{ + "about": "Accessibility violations the frontend has today, recorded so the scans pass on it and fail on a CHANGE (see README.md in this folder). Fixing one means deleting its entry here in the same pull request; adding one is not allowed except by a deliberate decision. 'nodes' is how many elements axe-core reported. Frontend fixes are tracked separately from #168 (frontend direction: #154).", + "violations": [ + { + "scenario": "upload_page_with_history", + "rule": "aria-progressbar-name", + "nodes": 3, + "note": "each import job's progress bar has no accessible name (index.tsx:392)" + }, + { + "scenario": "upload_page_with_history", + "rule": "button-name", + "nodes": 1, + "note": "a select (role=combobox) under the file list has no accessible name" + }, + { + "scenario": "upload_page_with_history", + "rule": "color-contrast", + "nodes": 4, + "note": "job cards: table-name chip 4.34:1, file size 4.35:1, failed and duplicate status pills 3.67:1 and 4.48:1; 4.5:1 is required (index.tsx:344,382,393)" + }, + { + "scenario": "upload_mapping_dialog", + "rule": "button-name", + "nodes": 2, + "note": "the column-type select in each row of the mapping dialog has no accessible name" + }, + { + "scenario": "upload_page_toast", + "rule": "button-name", + "nodes": 1, + "note": "a select (role=combobox) under the file list has no accessible name" + }, + { + "scenario": "upload_page_toast", + "rule": "color-contrast", + "nodes": 1, + "note": "the error toast's text (#e60000 on #fff0f0) is 4.34:1; 4.5:1 is required" + }, + { + "scenario": "audit_log_with_history", + "rule": "button-name", + "nodes": 2, + "note": "a select (role=combobox) and the icon-only per-row download button (audit.tsx:183) have no accessible name" + }, + { + "scenario": "audit_log_with_history", + "rule": "color-contrast", + "nodes": 2, + "note": "the '120 ok' count (#009966 on white) is 3.65:1 and the failed-status pill (#e7000b on pale red) is 4.0:1; 4.5:1 is required (audit.tsx:169,173)" + }, + { + "scenario": "audit_log_report_menu", + "rule": "aria-hidden-focus", + "nodes": 1, + "note": "opening the report menu marks the page aria-hidden while it still contains focusable controls (audit.tsx:92)" + }, + { + "scenario": "audit_log_report_menu", + "rule": "color-contrast", + "nodes": 2, + "note": "the same two contrast failures as audit_log_with_history, still on the page behind the open menu" + } + ] +} diff --git a/tests/accessibility/stub_api.py b/tests/accessibility/stub_api.py new file mode 100644 index 0000000..de8c89b --- /dev/null +++ b/tests/accessibility/stub_api.py @@ -0,0 +1,125 @@ +"""The API as the browser sees it during an accessibility scan (#168). + +The scans check the *frontend's* markup, so they need no database and no API process: +every ``/api/**`` request is answered here with a fixed response, and a signed-in session +is planted in sessionStorage. Same inputs, same pages, same result on every run, and a +backend-only pull request cannot change what is scanned. + +The payloads must satisfy the frontend's zod schemas (csv-table-hub-main/src/lib/ +csv.functions.ts); a schema mismatch shows up as a page that shows an error instead of the +content, which the scenarios' content checks catch. + +No Playwright import here: this module only describes data and a request handler. +""" +from __future__ import annotations + +import json +from typing import Any, Callable + +SESSION = {"token": "a11y-scan-token", "expiresAt": "2099-01-01T00:00:00Z", + "username": "ada", "tenant": "acme", "role": "admin"} + +FILES = [{ + "id": "1", "file_name": "customers.csv", "table_name": "csv_customers", "row_count": 120, + "total_rows": 125, "failed_rows": 5, "uploader": "ada", "upload_id": "101", + "column_names": ["id", "name", "email"], "created_at": "2026-11-01T01:02:03Z", +}] + +TE_TABLES = [{"table": "organisations", "exists": True, "rowCount": 3}, + {"table": "programs", "exists": True, "rowCount": 2}] + +# Import history, kept by the browser (localStorage), one job per kind of outcome. +_LOG = [{"step": "done", "level": "info", "message": "Imported 120 rows"}] +JOBS = [ + {"id": "j1", "batchId": "b1", "name": "customers.csv", "size": 2048, "status": "done", "progress": 100, + "createdAt": 1793500000000, "insertedRows": 120, "totalRows": 125, "failedRows": 5, + "tableName": "csv_customers", "columns": ["id", "name", "email"], "types": ["int8", "text", "text"], + "logs": _LOG}, + {"id": "j2", "batchId": "b1", "name": "orders.csv", "size": 100, "status": "error", "progress": 100, + "createdAt": 1793500100000, "errorMessage": "None of the 2 rows could be imported", + "rowErrors": [{"row": 1, "column": "date", "value": "27 Oct 26", "reason": "not a timestamp"}]}, + {"id": "j3", "batchId": "b1", "name": "dup.csv", "size": 100, "status": "duplicate", "progress": 100, + "createdAt": 1793500200000, "existingFileName": "customers.csv", "duplicateReason": "content", + "tableName": "csv_customers"}, +] + +REVIEW_PAGE = { + "fileName": "customers.csv", "columns": ["id", "name"], "types": ["int8", "text"], "total": 2, + "offset": 0, "limit": 50, "excludedRows": 0, "editedRows": 0, + "rows": [ + {"rowNumber": 1, "original": ["1", "Ada"], "current": ["1", "Ada"], "editedColumns": [], + "decision": "include", "decisionReason": None, "errors": []}, + {"rowNumber": 2, "original": ["x", "Bob"], "current": ["x", "Bob"], "editedColumns": [], + "decision": "include", "decisionReason": None, + "errors": [{"column": "id", "value": "x", "reason": '"x" is not a whole number'}]}, + ], +} + +PREVIEW = {"status": "ok", "headers": ["id", "name"], "columns": ["id", "name"], + "inferredTypes": ["int8", "text"], "sampleRows": [["1", "a"], ["2", "b"]], + "totalRowsApprox": 2, "teTableMatch": None} + +# route -> JSON body. A callable receives the request path and returns the body. +Responses = dict[str, Any] + + +def default_responses() -> Responses: + return { + "/api/csv/files": FILES, + "/api/te/tables": TE_TABLES, + "/api/csv/preview": PREVIEW, + "/api/auth/login": {"detail": "Invalid credentials"}, + } + + +def respond(path: str, responses: Responses) -> tuple[int, Any]: + """(status, body) for one request path.""" + if path in responses: + body = responses[path] + body = body(path) if callable(body) else body + # A login that is meant to fail is the only 401 in the app that stays on the page. + return (401, body) if path == "/api/auth/login" and not _is_session(body) else (200, body) + if path.startswith("/api/csv/uploads/") and path.endswith("/rows"): + return 200, REVIEW_PAGE + return 404, {"detail": f"a11y stub has no response for {path}"} + + +def _is_session(body: Any) -> bool: + return isinstance(body, dict) and "token" in body + + +def handler(responses: Responses, unstubbed: list[str]) -> Callable: + """A Playwright route handler. Paths with no stub are recorded in ``unstubbed``.""" + def handle(route, request): + path = request.url.split("?")[0] + path = path[path.index("/api/"):] if "/api/" in path else path + status, body = respond(path, responses) + if status == 404: + unstubbed.append(f"{request.method} {path}") + route.fulfill(status=status, content_type="application/json", body=json.dumps(body)) + return handle + + +def seed_script(session: dict | None, jobs: list[dict] | None) -> str: + """JavaScript run before the page's own scripts on every document. + + Plants the signed-in session and, when given, the import history. + + The history needs a guard. In development, React StrictMode runs the effects of + csv-table-hub-main/src/hooks/use-local-storage.ts twice: the first pass writes the + hook's empty default over the stored history, the second pass reads that back, so + stored history is never shown. (A frontend bug, outside this issue; production builds + do not double-run effects.) Without the guard the audit page would be scanned empty and + pass without checking a single row, so writes of "[]" to that key are dropped. + """ + parts = [] + if session is not None: + parts.append(f"sessionStorage.setItem('te.session', {json.dumps(json.dumps(session))});") + if jobs is not None: + parts.append( + "(() => { const KEY = 'csv-migrator:jobs:v1', set = Storage.prototype.setItem;" + " Storage.prototype.setItem = function (k, v) { if (k === KEY && v === '[]') return;" + " return set.call(this, k, v); };" + f" set.call(localStorage, KEY, {json.dumps(json.dumps(jobs))}); }})();" + ) + return "\n".join(parts) diff --git a/tests/accessibility/test_accessibility.py b/tests/accessibility/test_accessibility.py new file mode 100644 index 0000000..b57ed51 --- /dev/null +++ b/tests/accessibility/test_accessibility.py @@ -0,0 +1,111 @@ +"""WCAG 2.1 A and AA scans of the frontend's main pages, with axe-core in Chromium (#168). + +Run locally: python -m pytest tests/accessibility -v +Needs: pip install -r requirements-a11y.txt && python -m playwright install chromium + (cd csv-table-hub-main && npm ci) +In CI: the ``accessibility`` job of .github/workflows/quality-gate.yml, and only there; + the integration job runs with ``-m "not accessibility"``. + +Each test opens a page in a state worth checking, WAITS FOR CONTENT THAT PROVES THE PAGE +RENDERED (a redirect to the sign-in page or an error screen would otherwise scan clean), +and then scans. Results are judged against known_violations.json; see README.md in this +folder for what that means, and why scans of an unfixed frontend still pass today. +""" +from __future__ import annotations + +import pytest + +from tests.accessibility import stub_api + +CSV = {"name": "people.csv", "mimeType": "text/csv", "buffer": b"id,name\n1,a\n2,b\n"} +NOT_CSV = {"name": "notes.txt", "mimeType": "text/plain", "buffer": b"not a csv"} +TIMEOUT_MS = 15_000 + + +def _shows(page, text: str): + page.get_by_text(text, exact=False).first.wait_for(state="visible", timeout=TIMEOUT_MS) + + +def test_sign_in_page(open_page, scan): + """The sign-in page, before anyone is signed in.""" + page = open_page("/auth", signed_in=False) + _shows(page, "Sign in to CSV Migrator") + scan(page, "sign_in") + + +def test_sign_in_page_after_a_failed_attempt(open_page, scan): + """The sign-in page showing the server's refusal.""" + page = open_page("/auth", signed_in=False) + page.get_by_label("Tenant").fill("acme") + page.get_by_label("Username").fill("ada") + page.get_by_label("Password").fill("not-the-password") + page.get_by_role("button", name="Sign in").click() + _shows(page, "Invalid tenant, username or password") + scan(page, "sign_in_failed") + + +def test_upload_page_with_nothing_uploaded_yet(open_page, scan): + """The upload page for a new tenant: no migrated files and no import history.""" + page = open_page("/", responses={"/api/csv/files": []}) + _shows(page, "No files yet") + scan(page, "upload_page_empty") + + +def test_upload_page_with_files_and_import_history(open_page, scan): + """The upload page with a migrated file and one import of each kind (done, failed, duplicate).""" + page = open_page("/", jobs=stub_api.JOBS) + _shows(page, "orders.csv") + _shows(page, "dup.csv") + scan(page, "upload_page_with_history") + + +def test_upload_page_column_mapping_dialog(open_page, scan): + """The dialog that opens when a CSV is chosen: column names and the type of each.""" + page = open_page("/") + page.set_input_files("input[type=file]", files=[CSV]) + _shows(page, "Preview: people.csv") + scan(page, "upload_mapping_dialog") + + +def test_upload_page_rejection_toast(open_page, scan): + """The toast shown when a file that is not a CSV is chosen.""" + page = open_page("/") + page.set_input_files("input[type=file]", files=[NOT_CSV]) + _shows(page, "Please drop .csv files.") + scan(page, "upload_page_toast") + + +def test_audit_log_with_history(open_page, scan): + """The audit log listing one import of each kind.""" + page = open_page("/audit", jobs=stub_api.JOBS) + _shows(page, "orders.csv") + _shows(page, "dup.csv") + scan(page, "audit_log_with_history") + + +def test_audit_log_report_menu(open_page, scan): + """The audit log with its report-download menu open.""" + page = open_page("/audit", jobs=stub_api.JOBS) + _shows(page, "orders.csv") + page.get_by_role("button", name="Audit report").click() + page.get_by_role("menuitem").first.wait_for(state="visible", timeout=TIMEOUT_MS) + scan(page, "audit_log_report_menu") + + +def test_review_page(open_page, scan): + """The review screen for one stored upload, with a row that failed validation.""" + page = open_page("/review/101") + _shows(page, "Bob") + scan(page, "review_page") + + +@pytest.mark.parametrize("path", ["/", "/audit", "/review/101"]) +def test_signed_out_visitors_are_sent_to_sign_in(open_page, path): + """A page that needs a session sends a visitor with none to the sign-in page. + + This guards the other scenarios: a scan of a page that silently redirected would + be a scan of the sign-in page and pass for the wrong reason. + """ + page = open_page(path, signed_in=False) + page.wait_for_url("**/auth", timeout=TIMEOUT_MS) + _shows(page, "Sign in to CSV Migrator") diff --git a/tests/unit/test_a11y_baseline.py b/tests/unit/test_a11y_baseline.py new file mode 100644 index 0000000..a3be4a5 --- /dev/null +++ b/tests/unit/test_a11y_baseline.py @@ -0,0 +1,123 @@ +"""The accessibility baseline logic (#168): what counts as new, fixed and known. + +No browser: tests/accessibility/axe_baseline.py is pure. The scans themselves are in +tests/accessibility/test_accessibility.py and run only in the accessibility CI job. +""" +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +import pytest + +from tests.accessibility import axe_baseline as ab + +pytestmark = pytest.mark.unit + + +def _violation(rule: str, count: int, impact: str = "serious") -> dict: + return {"id": rule, "impact": impact, "help": f"help for {rule}", "helpUrl": f"https://axe/{rule}", + "nodes": [{"target": [f".n{i}"], "html": f"x"} for i in range(count)]} + + +def _entry(scenario: str, rule: str, nodes: int, note: str = "known") -> dict: + return {"scenario": scenario, "rule": rule, "nodes": nodes, "note": note} + + +def _baseline(*entries: dict) -> ab.Baseline: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "b.json" + path.write_text(json.dumps({"violations": list(entries)}), encoding="utf-8") + return ab.load_baseline(path) + + +class Compare(unittest.TestCase): + def test_nothing_found_and_nothing_recorded_is_ok(self): + out = ab.compare("page", [], _baseline()) + self.assertTrue(out.ok) + self.assertEqual((out.new, out.fixed, out.shrunk, out.known), ([], [], [], [])) + + def test_a_violation_not_in_the_baseline_is_new_and_fails(self): + out = ab.compare("page", [_violation("label", 1)], _baseline()) + self.assertFalse(out.ok) + self.assertEqual(len(out.new), 1) + self.assertIn("NEW label", out.new[0]) + + def test_the_baseline_is_per_scenario(self): + out = ab.compare("other", [_violation("label", 1)], _baseline(_entry("page", "label", 1))) + self.assertEqual(len(out.new), 1, "a rule known on one page is new on another") + self.assertEqual(out.fixed, [], "and the other page's entry is not this scenario's to judge") + + def test_exactly_as_recorded_is_known_and_passes(self): + out = ab.compare("page", [_violation("label", 2)], _baseline(_entry("page", "label", 2, "why"))) + self.assertTrue(out.ok) + self.assertEqual(len(out.known), 1) + self.assertIn("why", out.known[0]) + + def test_more_elements_than_recorded_is_a_regression(self): + out = ab.compare("page", [_violation("label", 3)], _baseline(_entry("page", "label", 2))) + self.assertFalse(out.ok) + self.assertIn("MORE ELEMENTS than the baseline's 2", out.new[0]) + + def test_fewer_elements_is_a_notice_not_a_failure(self): + out = ab.compare("page", [_violation("label", 1)], _baseline(_entry("page", "label", 2))) + self.assertTrue(out.ok) + self.assertEqual(len(out.shrunk), 1) + + def test_a_recorded_rule_that_no_longer_fails_must_be_removed(self): + out = ab.compare("page", [], _baseline(_entry("page", "label", 2, "was broken"))) + self.assertFalse(out.ok) + self.assertIn("delete its entry", out.fixed[0]) + self.assertIn("was broken", out.fixed[0]) + + def test_new_and_fixed_are_reported_together(self): + out = ab.compare("page", [_violation("region", 1)], _baseline(_entry("page", "label", 1))) + self.assertEqual((len(out.new), len(out.fixed)), (1, 1)) + + +class Describe(unittest.TestCase): + def test_lists_a_few_elements_and_counts_the_rest(self): + text = ab.describe(_violation("label", 5), limit=2) + self.assertIn("label (serious) on 5 element(s)", text) + self.assertIn(".n0", text) + self.assertIn(".n1", text) + self.assertNotIn(".n2", text) + self.assertIn("and 3 more", text) + self.assertIn("https://axe/label", text) + + +class LoadBaseline(unittest.TestCase): + def _load(self, *entries): + return _baseline(*entries) + + def test_rejects_a_rule_listed_twice_for_a_scenario(self): + with self.assertRaisesRegex(ValueError, "listed twice"): + self._load(_entry("p", "label", 1), _entry("p", "label", 2)) + + def test_rejects_an_entry_without_a_positive_count(self): + with self.assertRaisesRegex(ValueError, "positive integer"): + self._load(_entry("p", "label", 0)) + + def test_rejects_an_entry_without_a_note(self): + with self.assertRaisesRegex(ValueError, "note"): + self._load(_entry("p", "label", 1, note="")) + + +class RecordedBaseline(unittest.TestCase): + """known_violations.json itself: well formed, and about scenarios that exist.""" + + def test_is_valid_and_names_real_scenarios(self): + baseline = ab.load_baseline() + source = (Path(ab.__file__).with_name("test_accessibility.py")).read_text(encoding="utf-8") + for scenario, rule in baseline: + with self.subTest(scenario=scenario, rule=rule): + self.assertIn(f'"{scenario}"', source, "no scan uses this scenario name") + + def test_scans_wcag_21_a_and_aa(self): + self.assertEqual(ab.WCAG_21_AA_TAGS, ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_ci_quality_gate.py b/tests/unit/test_ci_quality_gate.py index 7d9df96..d1bcd54 100644 --- a/tests/unit/test_ci_quality_gate.py +++ b/tests/unit/test_ci_quality_gate.py @@ -93,6 +93,36 @@ def test_all_jobs_install_dev_deps(self): ) +class TestAccessibilityJob(unittest.TestCase): + """#168: the accessibility scans have their own job, and no other job collects them.""" + + @classmethod + def setUpClass(cls): + cls.jobs = _load_workflow().get("jobs", {}) + + def _run_text(self, job_id: str) -> str: + return "\n".join(s.get("run", "") for s in self.jobs[job_id].get("steps", [])) + + def test_has_accessibility_job_that_runs_only_the_accessibility_marker(self): + self.assertIn("accessibility", self.jobs) + self.assertIn('--markers accessibility', self._run_text("accessibility")) + + def test_accessibility_job_installs_a_browser_and_the_frontend(self): + text = self._run_text("accessibility") + self.assertIn("playwright install", text) + self.assertIn("npm ci", text) + self.assertIn("requirements-a11y.txt", text) + + def test_integration_job_excludes_accessibility_everywhere_it_runs_the_whole_tree(self): + text = self._run_text("integration-postgres") + self.assertIn('--markers "not accessibility"', text) + self.assertIn('pytest tests/ -m "not accessibility"', text) + + def test_free_tier_markers_cannot_select_accessibility_tests(self): + text = self._run_text("free-tier") + self.assertNotIn("accessibility", text.split("--markers", 1)[1].splitlines()[0]) + + class TestGapAnalysisG2Accepted(unittest.TestCase): """G2 is accepted as out-of-scope, not closed. Checks the decision is recorded in docs/GAP_ANALYSIS.md.