Repository navigation
Test layout, accessibility job, registry-race fix, ragged/XLSX test deltas (#167, #168, #191, #192, #193) #239
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Quality Gate | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - master | |
| pull_request: | |
| jobs: | |
| # Fast, deterministic checks — no database, no network beyond pip. | |
| free-tier: | |
| name: free-tier (unit, lint, health, evals-p) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements-dev.txt | |
| api/requirements.txt | |
| - name: Install dev dependencies | |
| # api/requirements.txt is needed too: tests/test_api.py imports FastAPI | |
| # at module level, so without it pytest collection fails. | |
| run: pip install -r requirements-dev.txt -r api/requirements.txt | |
| - name: Unit / regression / security / snapshot tests — final result | |
| run: | | |
| python3 scripts/test_report.py \ | |
| --markers "unit or regression or security or snapshot" --strict | |
| - name: Lint (flake8 + bandit) | |
| run: make lint | |
| - name: Component health check | |
| run: make health | |
| # #169: test catalogue freshness — fails if docs/TEST_CASES.md is stale. | |
| - name: Test catalogue check | |
| run: python3 scripts/test_catalogue.py --check | |
| # Tier X (cross-engine) is deliberately absent: multi-engine support is | |
| # deferred by ADR-0001 (PostgreSQL first), so this pipeline is PostgreSQL | |
| # only. The scenario still exists and passes — run it with `make | |
| # evals-engines`. Re-add `,x` here when multi-engine work resumes. | |
| - name: Evals — Tier P (offline validator scenarios) | |
| run: python3 evals/runner.py --tiers p --verbose | |
| # Reports (incl. the VCRM gap report) are written to evals/reports/ and | |
| # would otherwise vanish with the runner — keep them for auditability. | |
| - name: Upload eval reports | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: eval-reports-free-tier | |
| path: evals/reports/ | |
| if-no-files-found: ignore | |
| retention-days: 30 | |
| # Full integration surface: Tier I deploys env_dev.sql twice (idempotency), | |
| # Tier S runs the 142-assertion SQL suite, then the PG-gated pytest tiers run. | |
| integration-postgres: | |
| name: integration (postgres service) | |
| runs-on: ubuntu-latest | |
| services: | |
| postgres: | |
| image: postgres:16-alpine | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| PGHOST: localhost | |
| PGPORT: "5432" | |
| PGUSER: postgres | |
| PGPASSWORD: postgres | |
| PGDATABASE: postgres | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements-dev.txt | |
| api/requirements.txt | |
| - name: Install dev dependencies | |
| # api/requirements.txt is needed too: tests/test_api.py imports FastAPI | |
| # at module level, so without it pytest collection fails. | |
| run: pip install -r requirements-dev.txt -r api/requirements.txt | |
| - name: Install PostgreSQL client | |
| run: sudo apt-get update && sudo apt-get install -y --no-install-recommends postgresql-client | |
| - name: Write CI database config | |
| run: | | |
| cat > build/config.local.env <<'EOF' | |
| DB_ENGINE="postgresql" | |
| PG_HOST="localhost" | |
| PG_PORT="5432" | |
| PG_SUPERUSER="postgres" | |
| PG_SUPERUSER_PASSWORD="postgres" | |
| PG_DB_DEV="te_mgmt_dev" | |
| PG_SCHEMA_DEV="te_dev" | |
| PG_DB_TEST="te_mgmt_test" | |
| PG_SCHEMA_TEST="te_test" | |
| PG_DB_STAGING="te_mgmt_staging" | |
| PG_SCHEMA_STAGING="te_staging" | |
| PG_DB_PROD="te_mgmt_prod" | |
| PG_SCHEMA_PROD="te_prod" | |
| EOF | |
| # env_*.sql scripts do not create their databases — deploy_all.sh does that | |
| # before invoking them (see build/te_core_schema.sql:41). Mirror it here. | |
| # Only *.example.sql are committed (concrete env_<env>.sql are gitignored), | |
| # so materialise them before anything that deploys an environment. | |
| - name: Materialise environment launchers from templates | |
| run: | | |
| for env in dev test staging prod; do | |
| cp "build/environments/env_${env}.example.sql" \ | |
| "build/environments/env_${env}.sql" | |
| done | |
| ls -1 build/environments/ | |
| - name: Create environment databases | |
| run: | | |
| for db in te_mgmt_dev te_mgmt_test te_mgmt_staging te_mgmt_prod; do | |
| if [ -z "$(psql -d postgres -tA -c "SELECT 1 FROM pg_database WHERE datname = '$db'")" ]; then | |
| psql -v ON_ERROR_STOP=1 -d postgres -c \ | |
| "CREATE DATABASE \"$db\" WITH OWNER = postgres ENCODING = 'UTF8' TEMPLATE = template0 CONNECTION LIMIT = -1" | |
| fi | |
| done | |
| # Tier E deploys all four environments and compares their schemas, so it | |
| # runs here where the postgres service exists. Order matters: E deploys | |
| # every environment, which the cross-env parity pytest markers then rely on. | |
| # Tier X is excluded per ADR-0001 — see the free-tier job's note. | |
| - name: Evals — Tiers P, I, S, E | |
| run: python3 evals/runner.py --tiers p,i,s,e --verbose | |
| - name: Deploy all environments (for cross-env parity) | |
| run: | | |
| for env in dev test staging prod; do | |
| bash build/deploy_all.sh "$env" | |
| done | |
| # Prints a final block accounting for every test: PASSED / FAILED / | |
| # ERROR / SKIPPED (with reasons) / NOT RUN (deselected). --strict fails | |
| # the build on any skip. | |
| # The API reads PGDATABASE (api/config.py). The schema lives in | |
| # te_mgmt_dev, not the job-level default "postgres" (issue #45). | |
| # Accessibility tests need Playwright browsers and a running frontend, | |
| # neither of which exists in this job — they run in the `accessibility` | |
| # job below. Deselected, not skipped: test_report.py accounts for them | |
| # under NOT RUN. | |
| - name: Full test suite — final result with skip accounting | |
| env: | |
| PGDATABASE: te_mgmt_dev | |
| run: python3 scripts/test_report.py --markers "not accessibility" --strict | |
| # #194: measure api/services coverage with per-loader floors. | |
| # Accessibility tests need a browser and a running frontend (own job below). | |
| - name: api/services coverage gate | |
| env: | |
| PGDATABASE: te_mgmt_dev | |
| run: | | |
| python -m pytest tests/ -m "not accessibility" \ | |
| --cov=api/services \ | |
| --cov-report=json:coverage-services.json \ | |
| --cov-report=term-missing \ | |
| -q --no-header 2>&1 | tail -20 | |
| python3 scripts/check_coverage_floors.py coverage-services.json | |
| - name: Upload eval reports | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: eval-reports-integration | |
| path: evals/reports/ | |
| if-no-files-found: ignore | |
| retention-days: 30 | |
| # WCAG 2.1 AA scans (#168). Needs a browser and a running frontend, so it | |
| # cannot ride either job above. | |
| # | |
| # The scan step is continue-on-error ON PURPOSE: the frontend has known, | |
| # unfixed accessibility defects (sonner toast contrast, selects without | |
| # accessible names, download-menu aria-labels, upload progress bar), so the | |
| # scans fail today and that failure is the backlog, not a regression. The | |
| # report is uploaded as an artifact either way. Remove continue-on-error — | |
| # and only then add this job to branch protection — once the frontend fixes | |
| # land. | |
| accessibility: | |
| name: accessibility (WCAG 2.1 AA) | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Setup Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: | | |
| requirements-dev.txt | |
| api/requirements.txt | |
| - name: Install dev dependencies | |
| run: pip install -r requirements-dev.txt -r api/requirements.txt | |
| - name: Install Playwright Chromium | |
| run: python -m playwright install --with-deps chromium | |
| - name: Setup Node | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: '22' | |
| cache: 'npm' | |
| cache-dependency-path: csv-table-hub-main/package-lock.json | |
| # npm, not bun: bun.lock is out of step with package.json (zod ^3 vs ^4), | |
| # so `bun install --frozen-lockfile` fails; package-lock.json is current. | |
| - name: Install frontend dependencies | |
| working-directory: csv-table-hub-main | |
| run: npm ci --no-audit --no-fund | |
| - name: Start the Vite dev server | |
| working-directory: csv-table-hub-main | |
| run: | | |
| npx vite dev --host 127.0.0.1 --port 5173 --strictPort > "$RUNNER_TEMP/vite.log" 2>&1 & | |
| for i in $(seq 1 60); do | |
| if curl -fsS -o /dev/null http://127.0.0.1:5173/auth; then | |
| echo "dev server up after ${i}s"; exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "::error::dev server did not answer on 127.0.0.1:5173 within 60s" | |
| cat "$RUNNER_TEMP/vite.log" | |
| exit 1 | |
| - name: WCAG 2.1 AA scans | |
| continue-on-error: true | |
| env: | |
| A11Y_BASE_URL: http://127.0.0.1:5173 | |
| run: | | |
| python -m pytest tests/accessibility -v \ | |
| --junitxml="$RUNNER_TEMP/accessibility.xml" | |
| - name: Upload accessibility results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: accessibility-results | |
| path: | | |
| ${{ runner.temp }}/accessibility.xml | |
| ${{ runner.temp }}/vite.log | |
| if-no-files-found: ignore | |
| retention-days: 30 |