diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b84faa..e859bc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,13 @@ on: push: branches: [main] pull_request: + # Nothing here pins its dependencies beyond uv.lock's resolution, so a + # transitive release can break the build with no commit of ours involved. + # Weekly is often enough to find that while it is still a small problem, and + # rare enough that nobody learns to ignore the mail. + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: # No secrets are used anywhere in this workflow, and none should be added. # Every test must pass with no network connection and no API key. A test that @@ -57,3 +64,112 @@ jobs: - name: Tests run: uv run pytest + + # The unit suite has twice now passed while the path a new user takes first + # was broken: an adapter that raised a bare TypeError when a setting was + # missing, and a local arm that failed every case with the reason visible + # only inside the artifact. Both were found by hand. This job walks the + # README quickstart instead of trusting it. + quickstart: + name: quickstart, as the README documents it + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Sync dependencies + run: uv sync --locked + + - name: Run the quickstart command verbatim + run: | + uv run plumbline run datasets/public/jevbench-hard.jsonl \ + --adapter mock --format jevbench \ + --results results --report results/report.md + + - name: The report says what it is supposed to say + run: | + test -f results/report.md || { echo "no report written"; exit 1; } + grep -q "ECE" results/report.md + # The inconclusive verdict must not regress to reading as a pass. + grep -q "INCONCLUSIVE" results/report.md + grep -q '"INCONCLUSIVE" is not a pass' results/report.md + # Every calibration figure carries its row count and its floor. + grep -q "calibrated-model floor" results/report.md + + - name: Following the quickstart leaves the clone clean + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "the quickstart dirtied the working tree:" + git status --porcelain + exit 1 + fi + + - name: The other documented commands work + run: | + uv run plumbline --help > /dev/null + uv run plumbline adapters | grep -q mock + test "$(uv run plumbline version)" = "$(uv run python -c 'import plumbline; print(plumbline.__version__)')" + + - name: A missing required setting refuses instead of crashing + run: | + set +e + out="$(uv run plumbline run datasets/public/jevbench-hard.jsonl \ + --adapter local_logits --format jevbench --limit 1 \ + --results results 2>&1)" + set -e + echo "$out" | grep -q "requires" || { echo "expected a named refusal, got:"; echo "$out"; exit 1; } + echo "$out" | grep -qv "Traceback" || { echo "traceback leaked"; exit 1; } + + # Gate 6 found that the built distribution is a different artifact from the + # source tree: the entry point, the optional extra boundary, and py.typed all + # only exist once packaged. + wheel: + name: the built wheel installs and runs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Build + run: uv build + + - name: Install the wheel alone, into a clean environment + run: | + uv venv --python 3.13 /tmp/venv + uv pip install --python /tmp/venv/bin/python dist/*.whl + + - name: The entry point resolves and the package is importable + run: | + /tmp/venv/bin/plumbline version + /tmp/venv/bin/plumbline adapters | grep -q typesafe_wire + /tmp/venv/bin/python -c "import plumbline" + + - name: The local extra is genuinely optional + run: | + /tmp/venv/bin/python - <<'PY' + import importlib.util as u + for name in ("torch", "transformers"): + assert u.find_spec(name) is None, f"{name} leaked into the core install" + print("core install carries neither torch nor transformers") + PY + + - name: py.typed ships, so downstream annotations are visible + run: | + /tmp/venv/bin/python - <<'PY' + import pathlib, plumbline + marker = pathlib.Path(plumbline.__file__).parent / "py.typed" + assert marker.is_file(), "py.typed missing from the installed package" + print("py.typed present at", marker) + PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d9ab2ae --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,105 @@ +name: Release + +# A release is a tag push. Nothing here is built by hand on a maintainer's +# laptop, because the v0.1.0 wheel was, and it went out named +# plumbline-0.1.0.dev0 against a tag that said v0.1.0. +on: + push: + tags: ["v*"] + # So a release can be re-run against an existing tag without moving it. + workflow_dispatch: + inputs: + tag: + description: "Existing tag to build and attach, e.g. v0.1.0" + required: true + type: string + +permissions: + contents: write # attaching assets to the release + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + name: build and attach the distributions + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.tag || github.ref }} + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Sync dependencies + run: uv sync --locked + + # The whole reason this workflow exists. A tag and a version that + # disagree is exactly what shipped in v0.1.0, and it was invisible + # because the test asserting the version used a substring match. + - name: The tag and the package version must agree + run: | + tag="${{ inputs.tag || github.ref_name }}" + expected="${tag#v}" + actual="$(uv run python -c 'import plumbline; print(plumbline.__version__)')" + echo "tag $tag implies version $expected; package says $actual" + if [ "$expected" != "$actual" ]; then + echo "::error::tag $tag does not match __version__ $actual. Bump the version or move the tag; do not release a mismatch." + exit 1 + fi + + - name: The gate still passes at this tag + run: | + uv run ruff check . + uv run ruff format --check . + uv run mypy --strict + uv run pytest -q + + - name: Build the wheel and the sdist + run: uv build + + - name: The artifact names carry the released version + run: | + tag="${{ inputs.tag || github.ref_name }}" + version="${tag#v}" + ls -l dist/ + test -f "dist/plumbline-${version}-py3-none-any.whl" \ + || { echo "::error::expected dist/plumbline-${version}-py3-none-any.whl"; exit 1; } + test -f "dist/plumbline-${version}.tar.gz" \ + || { echo "::error::expected dist/plumbline-${version}.tar.gz"; exit 1; } + + - name: The wheel installs clean, with neither torch nor transformers + run: | + uv venv --python 3.13 /tmp/relvenv + uv pip install --python /tmp/relvenv/bin/python dist/*.whl + /tmp/relvenv/bin/plumbline version + /tmp/relvenv/bin/python - <<'PY' + import importlib.util as u, pathlib, plumbline + for name in ("torch", "transformers"): + assert u.find_spec(name) is None, f"{name} leaked into the core install" + assert (pathlib.Path(plumbline.__file__).parent / "py.typed").is_file() + print("installed", plumbline.__version__, "clean") + PY + + - name: Attach the distributions to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tag="${{ inputs.tag || github.ref_name }}" + # The release notes are written by hand, so this only ever adds + # assets to a release that already exists. It does not create one and + # it does not generate notes from commit messages. + if ! gh release view "$tag" > /dev/null 2>&1; then + echo "::error::no release exists for $tag. Create it with written notes first, then re-run this workflow." + exit 1 + fi + gh release upload "$tag" dist/* --clobber + echo "attached:" + gh release view "$tag" --json assets --jq '.assets[].name' diff --git a/docs/PLAN.md b/docs/PLAN.md index 1197669..a8a6673 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -8,9 +8,11 @@ this page and pick up where the work stopped. landed on 2026-09-21. CI is green on Ubuntu and Windows across Python 3.12 and 3.13. `v0.1.0` is tagged and released. -What remains is a short list of manual steps for a human, in "Pre-public -checklist" at the bottom of this page. The repository is still private, and -flipping it is a deliberate act that nothing here does for you. +The repository is public as of 2026-09-21. The "Pre-public checklist" at the +bottom of this page is kept as the record of what was verified before that +happened, not as outstanding work. + +The remaining work is the v0.2 milestone, and #3 comes first. ## Phases