ci.yml

name: CI

# A branch pushed to origin and then opened as a pull request fired both
# triggers, so every pull request ran all four jobs twice. The push trigger is
# restricted to main, which is the only branch where a push is not already
# covered by a pull request. pull_request stays unfiltered so a request from
# any branch or fork is checked.
on:
  push:
    branches: [main]
  pull_request:
  # Nothing here pins its dependencies beyond uv.lock's resolution, so a
  # transitive release can break the build with no commit of ours involved.
  # Weekly is often enough to find that while it is still a small problem, and
  # rare enough that nobody learns to ignore the mail.
  schedule:
    - cron: "17 6 * * 1"
  workflow_dispatch:

# No secrets are used anywhere in this workflow, and none should be added.
# Every test must pass with no network connection and no API key. A test that
# needs either is wrong and gets marked and excluded, not accommodated with a
# secret.
permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  check:
    name: ${{ matrix.os }} / Python ${{ matrix.python }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        # Every platform and every Python that requires-python admits and
        # that has a release, so ">=3.12" states what is tested.
        os: [ubuntu-latest, windows-latest, macos-latest]
        python: ["3.12", "3.13", "3.14"]

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          python-version: ${{ matrix.python }}
          enable-cache: true

      - name: Sync dependencies
        # The local extra (torch, transformers) is deliberately not installed.
        # plumbline must import and run hosted-only, and CI is where that stays
        # true.
        run: uv sync --locked

      - name: Lint
        run: uv run ruff check .

      - name: Format
        run: uv run ruff format --check .

      - name: Types
        run: uv run mypy --strict

      - name: Tests
        run: uv run pytest

  # uv.lock pins the newest releases, so the matrix never sees the oldest ones
  # pyproject still admits. The floors were once set by hand and were wrong for
  # three packages at once. This job installs the oldest direct dependency each
  # floor allows and runs the suite on it, so a floor that no longer holds fails
  # here, not on a user's machine.
  lowest:
    name: the oldest dependencies pyproject allows
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          python-version: "3.12"

      - name: Resolve the lowest direct dependencies
        # uv sync alone keeps a lockfile that already satisfies pyproject, so
        # the lowest resolution is locked first and then installed as written.
        run: uv lock --resolution lowest-direct

      - name: Sync dependencies
        run: uv sync --frozen

      - name: Tests
        run: uv run --frozen pytest

  # The unit suite has twice now passed while the path a new user takes first
  # was broken: an adapter that raised a bare TypeError when a setting was
  # missing, and a local arm that failed every case with the reason visible
  # only inside the artifact. Both were found by hand. This job walks the
  # README quickstart instead of trusting it.
  quickstart:
    name: quickstart, as the README documents it
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          python-version: "3.13"
          enable-cache: true

      - name: Sync dependencies
        run: uv sync --locked

      - name: Run the quickstart command verbatim
        run: |
          uv run plumbline run datasets/public/jevbench-hard.jsonl \
              --adapter mock --format jevbench \
              --results results --report results/report.md

      - name: The report says what it is supposed to say
        run: |
          test -f results/report.md || { echo "no report written"; exit 1; }
          grep -q "ECE" results/report.md
          # The inconclusive verdict must not regress to reading as a pass.
          grep -q "INCONCLUSIVE" results/report.md
          grep -q '"INCONCLUSIVE" is not a pass' results/report.md
          # Every calibration figure carries its row count and its floor.
          grep -q "calibrated-model floor" results/report.md

      - name: Following the quickstart leaves the clone clean
        run: |
          if [ -n "$(git status --porcelain)" ]; then
            echo "the quickstart dirtied the working tree:"
            git status --porcelain
            exit 1
          fi

      - name: The other documented commands work
        run: |
          uv run plumbline --help > /dev/null
          uv run plumbline adapters | grep -q mock
          test "$(uv run plumbline version)" = "$(uv run python -c 'import plumbline; print(plumbline.__version__)')"

      - name: A missing required setting refuses instead of crashing
        run: |
          set +e
          out="$(uv run plumbline run datasets/public/jevbench-hard.jsonl \
                   --adapter local_logits --format jevbench --limit 1 \
                   --results results 2>&1)"
          set -e
          echo "$out" | grep -q "requires" || { echo "expected a named refusal, got:"; echo "$out"; exit 1; }
          if echo "$out" | grep -q "Traceback"; then echo "traceback leaked"; echo "$out"; exit 1; fi

  # The house style has no em dashes, and the report's own strings are prose a
  # reader sees, as are the site's pages and the issue templates. Besides the
  # character itself it catches the spellings that render as one in HTML or JS
  # (the named and numeric HTML entities, and the JS unicode escape); each is
  # written in the pattern with a bracketed character so this file does not
  # match itself. datasets/public/ holds vendored third-party rows reproduced as
  # published, and site/vendor/ a vendored library, so both are excluded.
  prose:
    name: no em dashes in the docs or the report strings
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Tracked prose, source, site, and templates contain no em dash
        run: |
          pattern='\x{2014}|&m[d]ash;|&#821[2];|&#x201[4];|\\u201[4]'
          if git grep -n -i -P "$pattern" -- '*.md' 'src/' 'site/' 'scripts/' '.github/' \
              ':!datasets/public/' ':!site/vendor/'; then
            echo "::error::em dash found above; use a comma, colon, parentheses, or a new sentence"
            exit 1
          fi

      # -- is for command lines (flags, and git's end-of-options), never a stand-in
      # for a dash in prose. This finds a bare -- with words on both sides, or one
      # opening a continuation line; a flag like --adapter has no space after it,
      # and HTML comments and table rules have no word before it.
      - name: Prose uses real punctuation, not -- as a dash
        run: |
          pattern='[\w,.;:)\]]\s+--\s+[A-Za-z(]|^\s*--\s+[a-z]'
          if git grep -n -P "$pattern" -- '*.md' 'src/' 'tests/' 'examples/' 'conftest.py' \
              'site/' 'scripts/' ':!datasets/public/' ':!site/vendor/'; then
            echo "::error::-- used as a dash above; use a comma, colon, parentheses, or a new sentence"
            exit 1
          fi

  # Gate 6 found that the built distribution is a different artifact from the
  # source tree: the entry point, the optional extra boundary, and py.typed all
  # only exist once packaged.
  wheel:
    name: the built wheel installs and runs
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

      - name: Install uv
        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
        with:
          python-version: "3.13"
          enable-cache: true

      - name: Build
        run: uv build

      - name: Install the wheel alone, into a clean environment
        run: |
          uv venv --python 3.13 /tmp/venv
          uv pip install --python /tmp/venv/bin/python dist/*.whl

      - name: The entry point resolves and the package is importable
        run: |
          /tmp/venv/bin/plumbline version
          /tmp/venv/bin/plumbline adapters | grep -q typesafe_wire
          /tmp/venv/bin/python -c "import plumbline"

      - name: The local extra is genuinely optional
        run: |
          /tmp/venv/bin/python - <<'PY'
          import importlib.util as u
          for name in ("torch", "transformers"):
              assert u.find_spec(name) is None, f"{name} leaked into the core install"
          print("core install carries neither torch nor transformers")
          PY

      - name: py.typed ships, so downstream annotations are visible
        run: |
          /tmp/venv/bin/python - <<'PY'
          import pathlib, plumbline
          marker = pathlib.Path(plumbline.__file__).parent / "py.typed"
          assert marker.is_file(), "py.typed missing from the installed package"
          print("py.typed present at", marker)
          PY