From 4b7511985e6c050275fd929b13f72aa7d9407733 Mon Sep 17 00:00:00 2001 From: TMHSDigital <154358121+TMHSDigital@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:36:36 -0400 Subject: [PATCH] ci: guard the release and the path a new user takes first Three additions, each aimed at a failure that has actually happened here. A release workflow, on tag push. v0.1.0 was built by hand and went out as plumbline-0.1.0.dev0 against a tag reading v0.1.0, so the first thing this does is refuse when the tag and __version__ disagree. It then re-runs the gate at the tag, builds the wheel and sdist, asserts the filenames carry the released version, installs the wheel into a clean environment to confirm neither torch nor transformers leaked into the core install and that py.typed is present, and attaches both artifacts. It will not create a release and will not generate notes from commit messages: the notes are written, so the workflow refuses if no release exists for the tag. A quickstart job that walks the README rather than trusting it. The unit suite has twice passed while the first thing a new user does was broken: an adapter raising a bare TypeError on a missing setting, and a local arm failing every case with the reason visible only inside the artifact JSON. Both were found by hand. The job runs the documented command, asserts the report carries ECE, its floor, and the inconclusive wording that must not regress to reading as a pass, checks the clone is still clean afterwards, and confirms a missing required setting produces a named refusal with no traceback. A wheel job, because the built distribution is a different artifact from the source tree. The entry point, the optional-extra boundary and py.typed only exist once packaged, which is where Gate 6 found bugs. Also a weekly schedule and workflow_dispatch on CI. Nothing here pins transitive dependencies beyond uv.lock's resolution, so a release elsewhere can break the build with no commit of ours involved. Weekly finds that while it is small and is rare enough that nobody learns to ignore it. CI caching was already enabled via setup-uv, so nothing was added for it. PLAN.md no longer says the repository is private. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 116 ++++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 105 ++++++++++++++++++++++++++++++ docs/PLAN.md | 8 ++- 3 files changed, 226 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b84faa..e859bc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,13 @@ on: push: branches: [main] pull_request: + # Nothing here pins its dependencies beyond uv.lock's resolution, so a + # transitive release can break the build with no commit of ours involved. + # Weekly is often enough to find that while it is still a small problem, and + # rare enough that nobody learns to ignore the mail. + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: # No secrets are used anywhere in this workflow, and none should be added. # Every test must pass with no network connection and no API key. A test that @@ -57,3 +64,112 @@ jobs: - name: Tests run: uv run pytest + + # The unit suite has twice now passed while the path a new user takes first + # was broken: an adapter that raised a bare TypeError when a setting was + # missing, and a local arm that failed every case with the reason visible + # only inside the artifact. Both were found by hand. This job walks the + # README quickstart instead of trusting it. + quickstart: + name: quickstart, as the README documents it + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Sync dependencies + run: uv sync --locked + + - name: Run the quickstart command verbatim + run: | + uv run plumbline run datasets/public/jevbench-hard.jsonl \ + --adapter mock --format jevbench \ + --results results --report results/report.md + + - name: The report says what it is supposed to say + run: | + test -f results/report.md || { echo "no report written"; exit 1; } + grep -q "ECE" results/report.md + # The inconclusive verdict must not regress to reading as a pass. + grep -q "INCONCLUSIVE" results/report.md + grep -q '"INCONCLUSIVE" is not a pass' results/report.md + # Every calibration figure carries its row count and its floor. + grep -q "calibrated-model floor" results/report.md + + - name: Following the quickstart leaves the clone clean + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "the quickstart dirtied the working tree:" + git status --porcelain + exit 1 + fi + + - name: The other documented commands work + run: | + uv run plumbline --help > /dev/null + uv run plumbline adapters | grep -q mock + test "$(uv run plumbline version)" = "$(uv run python -c 'import plumbline; print(plumbline.__version__)')" + + - name: A missing required setting refuses instead of crashing + run: | + set +e + out="$(uv run plumbline run datasets/public/jevbench-hard.jsonl \ + --adapter local_logits --format jevbench --limit 1 \ + --results results 2>&1)" + set -e + echo "$out" | grep -q "requires" || { echo "expected a named refusal, got:"; echo "$out"; exit 1; } + echo "$out" | grep -qv "Traceback" || { echo "traceback leaked"; exit 1; } + + # Gate 6 found that the built distribution is a different artifact from the + # source tree: the entry point, the optional extra boundary, and py.typed all + # only exist once packaged. + wheel: + name: the built wheel installs and runs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Build + run: uv build + + - name: Install the wheel alone, into a clean environment + run: | + uv venv --python 3.13 /tmp/venv + uv pip install --python /tmp/venv/bin/python dist/*.whl + + - name: The entry point resolves and the package is importable + run: | + /tmp/venv/bin/plumbline version + /tmp/venv/bin/plumbline adapters | grep -q typesafe_wire + /tmp/venv/bin/python -c "import plumbline" + + - name: The local extra is genuinely optional + run: | + /tmp/venv/bin/python - <<'PY' + import importlib.util as u + for name in ("torch", "transformers"): + assert u.find_spec(name) is None, f"{name} leaked into the core install" + print("core install carries neither torch nor transformers") + PY + + - name: py.typed ships, so downstream annotations are visible + run: | + /tmp/venv/bin/python - <<'PY' + import pathlib, plumbline + marker = pathlib.Path(plumbline.__file__).parent / "py.typed" + assert marker.is_file(), "py.typed missing from the installed package" + print("py.typed present at", marker) + PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..d9ab2ae --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,105 @@ +name: Release + +# A release is a tag push. Nothing here is built by hand on a maintainer's +# laptop, because the v0.1.0 wheel was, and it went out named +# plumbline-0.1.0.dev0 against a tag that said v0.1.0. +on: + push: + tags: ["v*"] + # So a release can be re-run against an existing tag without moving it. + workflow_dispatch: + inputs: + tag: + description: "Existing tag to build and attach, e.g. v0.1.0" + required: true + type: string + +permissions: + contents: write # attaching assets to the release + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + name: build and attach the distributions + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ inputs.tag || github.ref }} + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + python-version: "3.13" + enable-cache: true + + - name: Sync dependencies + run: uv sync --locked + + # The whole reason this workflow exists. A tag and a version that + # disagree is exactly what shipped in v0.1.0, and it was invisible + # because the test asserting the version used a substring match. + - name: The tag and the package version must agree + run: | + tag="${{ inputs.tag || github.ref_name }}" + expected="${tag#v}" + actual="$(uv run python -c 'import plumbline; print(plumbline.__version__)')" + echo "tag $tag implies version $expected; package says $actual" + if [ "$expected" != "$actual" ]; then + echo "::error::tag $tag does not match __version__ $actual. Bump the version or move the tag; do not release a mismatch." + exit 1 + fi + + - name: The gate still passes at this tag + run: | + uv run ruff check . + uv run ruff format --check . + uv run mypy --strict + uv run pytest -q + + - name: Build the wheel and the sdist + run: uv build + + - name: The artifact names carry the released version + run: | + tag="${{ inputs.tag || github.ref_name }}" + version="${tag#v}" + ls -l dist/ + test -f "dist/plumbline-${version}-py3-none-any.whl" \ + || { echo "::error::expected dist/plumbline-${version}-py3-none-any.whl"; exit 1; } + test -f "dist/plumbline-${version}.tar.gz" \ + || { echo "::error::expected dist/plumbline-${version}.tar.gz"; exit 1; } + + - name: The wheel installs clean, with neither torch nor transformers + run: | + uv venv --python 3.13 /tmp/relvenv + uv pip install --python /tmp/relvenv/bin/python dist/*.whl + /tmp/relvenv/bin/plumbline version + /tmp/relvenv/bin/python - <<'PY' + import importlib.util as u, pathlib, plumbline + for name in ("torch", "transformers"): + assert u.find_spec(name) is None, f"{name} leaked into the core install" + assert (pathlib.Path(plumbline.__file__).parent / "py.typed").is_file() + print("installed", plumbline.__version__, "clean") + PY + + - name: Attach the distributions to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + tag="${{ inputs.tag || github.ref_name }}" + # The release notes are written by hand, so this only ever adds + # assets to a release that already exists. It does not create one and + # it does not generate notes from commit messages. + if ! gh release view "$tag" > /dev/null 2>&1; then + echo "::error::no release exists for $tag. Create it with written notes first, then re-run this workflow." + exit 1 + fi + gh release upload "$tag" dist/* --clobber + echo "attached:" + gh release view "$tag" --json assets --jq '.assets[].name' diff --git a/docs/PLAN.md b/docs/PLAN.md index 1197669..a8a6673 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -8,9 +8,11 @@ this page and pick up where the work stopped. landed on 2026-09-21. CI is green on Ubuntu and Windows across Python 3.12 and 3.13. `v0.1.0` is tagged and released. -What remains is a short list of manual steps for a human, in "Pre-public -checklist" at the bottom of this page. The repository is still private, and -flipping it is a deliberate act that nothing here does for you. +The repository is public as of 2026-09-21. The "Pre-public checklist" at the +bottom of this page is kept as the record of what was verified before that +happened, not as outstanding work. + +The remaining work is the v0.2 milestone, and #3 comes first. ## Phases