From d41c3c792c30e2ded776259e28a537b5d9eb5f8f Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 4 Aug 2026 14:27:36 -0700 Subject: [PATCH 1/9] ci(release): promote v0 only after PyPI publish, verify the published action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consumers pin `UiPath/coder_eval@v0`, and the composite action installs `coder-eval==`. The release job moved `v0` and cut the GitHub Release *before* the wheel was on PyPI, so a failure after the tag move stranded `@v0` on a pin that cannot resolve — `uv tool install` 404s and every consumer's pipeline breaks. It was reachable two ways: publish-pypi is a separate `needs: release` job that can fail or wait on the `pypi` environment gate, and the tag move sat before "Build wheel + sdist", so a build failure stranded the pin with PyPI never involved. Prevention, not detection: - Move the `v0` promotion and the GitHub Release into a new `promote` job gated on `needs: [release, publish-pypi]`. Nothing consumer-visible happens until the wheel is published. - `promote` is idempotent (force-push tag move, existence-guarded release create), so a failure is recovered by re-running the failed jobs — unlike the `release` job, which would bump a second version. That is what lets these steps fail loudly and removes the `continue-on-error` + annotation dance that existed only because a failure would have skipped publish-pypi. Detection, for what ordering cannot cover (a yank, a rename, a delisting): - New `verify-published-action.yml`. Tier 1 is free and deterministic: assert the major tag points at the newest release, that action.yml *at that tag* pins that version, that the version is on PyPI (retried for index propagation), that the Marketplace listing resolves, and that the wheel installs. Tier 2 consumes the action as a stranger would — `@v0`, default `version:`, no repo checkout, task YAML written inline. - Triggered on Release completion regardless of conclusion: a failed publish-pypi makes the run conclusion `failure`, so gating on success would skip the check exactly when it matters. Plus a daily cron and dispatch. - The e2e gate is ARTIFACTS, not the step's exit code. action.yml exits with coder-eval's own code, and coder-eval exits 1 on any failed task, so `minimum-task-score: 0.0` does not stop a model flake from reddening the build. It asserts run.json, a parseable JUnit, wired outputs, and non-zero tokens — "does the published action work", not "is the model still good". Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release.yml | 268 ++++++++------- .github/workflows/verify-published-action.yml | 305 ++++++++++++++++++ 2 files changed, 462 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/verify-published-action.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3ce0a578..c49aba7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,6 +60,10 @@ jobs: # Exposed so the downstream publish-pypi job gates on a version having been # produced (real release on main, or a stamped prerelease on a branch). version: ${{ steps.ver.outputs.version }} + # Real-release-only version: empty on a prerelease dispatch (the `release` + # step is skipped off main). The `promote` job gates on this, so a + # prerelease never moves the major tag or cuts a GitHub Release. + released_version: ${{ steps.release.outputs.version }} env: # Load-bearing on the release path: the pool enforces a package-age safe-chain # check on uv installs. Same expression as pr-checks.yml (see the comment there), @@ -230,18 +234,11 @@ jobs: VERSION: ${{ steps.release.outputs.version }} run: git push origin main "v${VERSION}" - - name: Move major action tag (vN -> this release) - if: steps.release.outputs.version != '' - env: - VERSION: ${{ steps.release.outputs.version }} - run: | - set -euo pipefail - # Consumers pin `UiPath/coder_eval@v0` (becomes `@v1` at 1.0.0). Force-move - # the moving major tag to this release. Force on a missing tag creates it. - MAJOR="v${VERSION%%.*}" - git tag -f "$MAJOR" "v${VERSION}" - git push -f origin "$MAJOR" - + # NOTE: the moving major tag (`v0`) is deliberately NOT moved here. It is the + # ref every consumer pins, and moving it before the wheel is on PyPI strands + # `@v0` on an action.yml pin whose version does not exist -- see the `promote` + # job at the bottom of this file, which moves it only after publish-pypi + # succeeds. - name: Build wheel + sdist if: steps.ver.outputs.version != '' run: uv build @@ -257,105 +254,11 @@ jobs: path: dist/ if-no-files-found: error - # Publish the GitHub Release for the tag pushed above. semantic-release runs - # with --no-vcs-release because it also runs --no-push (the commit is amended - # and the tag re-pointed first), so it cannot create the release itself -- it - # happens here, once the tag is actually on the remote. A published Release is - # what GitHub Marketplace listings are cut from, so every release needs one. - # (`gh release create` cannot tick the "Publish this Action to the - # Marketplace" checkbox -- that stays a one-time manual step in the GitHub UI - # on the first Release; every subsequent release then lists automatically.) - # - # Deliberately placed AFTER "Build wheel + sdist" and "Upload dist for PyPI - # publish" rather than at the earliest legal point after the tag push: those - # two steps are the last ones that can still fail for an already-tagged - # version, and a Release announcing a version whose artifacts never built is - # worse than a missing Release. This narrows the window rather than closing - # it -- publish-pypi is a separate job, so the actual upload to PyPI still - # happens after this. Running here also keeps a slow/hung `gh` API call from - # eating the 15-minute job budget BEFORE the artifacts are safe, which would - # produce exactly the stranded-tag state the note below warns about. - # - # Notes are the CHANGELOG section semantic-release just generated for this - # version, sliced by .github/scripts/release_notes.py (a real module, so the - # regex is unit-tested -- see tests/test_release_notes.py); an empty result - # falls back to GitHub's generated notes. - # - # ACCEPTED RISK: those notes render commit subjects, i.e. squashed PR titles. - # The Release body is a first-party surface that GitHub also fans out in - # notification emails, so it carries text that was reviewed as *code*, not as - # markdown -- a PR title can land an arbitrary link in it. Bounded to - # content/link spoofing (GitHub strips raw HTML from release bodies) and - # gated by this repo's mandatory PR review. Revisit with `--draft` plus a - # human glance, or link-stripping in release_notes.py, if the repo ever takes - # drive-by contributions. - - name: Publish GitHub Release - id: gh_release - if: steps.mode.outputs.prerelease != 'true' && steps.release.outputs.version != '' - # Best-effort, mirroring the GHCR steps below. main, the version tag, the - # moving major tag, and the dist artifact are all in place by the time this - # runs, so a transient GitHub API failure here must not fail the job: the - # publish-pypi job is `needs: release`, so a failure would SKIP the PyPI - # publish of an already-tagged version and strand `@vN` on an action.yml pin - # whose version was never published. The next step turns the swallowed - # failure into a loud annotation instead of a collapsed step marker. - continue-on-error: true - env: - # The app token, not GITHUB_TOKEN: the workflow's `permissions:` are - # contents: read, and `gh release create` needs contents: write. Granting - # the job contents: write to use GITHUB_TOKEN here would ADD a second - # write credential rather than remove one -- `actions/checkout` above - # already persists this same app token in .git/config for every step in - # the job, so scoping it out of this one step's env buys no isolation. - GH_TOKEN: ${{ steps.app-token.outputs.token }} - # Passed via env (not interpolated into the script) per GitHub's - # injection guidance. - VERSION: ${{ steps.release.outputs.version }} - run: | - set -euo pipefail - # Written under RUNNER_TEMP, never the repo root: hatchling's default sdist - # file selection sweeps in untracked files at the root (verified -- it ships - # even git-ignored paths), so a notes file left in the tree would leak into - # the sdist that the "Build wheel + sdist" step above produced and the - # publish-pypi job uploads. - NOTES_FILE="${RUNNER_TEMP}/release-notes.md" - python3 .github/scripts/release_notes.py "$VERSION" "$NOTES_FILE" - # Empty notes file => no CHANGELOG section was found (the script already - # emitted the ::warning::); let GitHub generate the body instead. - if [ -s "$NOTES_FILE" ]; then - NOTES=(--notes-file "$NOTES_FILE") - else - NOTES=(--generate-notes) - fi - gh release create "v${VERSION}" \ - --title "v${VERSION}" \ - --verify-tag \ - --latest \ - "${NOTES[@]}" - - # `continue-on-error` above hides a failure in a collapsed step marker that - # nobody expands on an otherwise-green release run -- the same silence that - # let "no GitHub Releases at all" go unnoticed until this PR. Re-raise it as - # an ::error annotation plus a run-summary block, WITHOUT failing the job - # (that would skip publish-pypi, see above). - - name: Flag missing GitHub Release - if: always() && steps.gh_release.outcome == 'failure' - env: - VERSION: ${{ steps.release.outputs.version }} - run: | - set -euo pipefail - MAJOR="v${VERSION%%.*}" - echo "::error title=GitHub Release not published::v${VERSION} was tagged and its artifacts built, but 'gh release create' failed. Create the Release by hand so ${MAJOR} and the Marketplace listing resolve." - { - echo "### :x: GitHub Release for \`v${VERSION}\` was NOT created" - echo - echo "The version tag, the moving \`${MAJOR}\` tag, and the PyPI artifacts are unaffected —" - echo "only \`gh release create\` failed. Create it by hand:" - echo - echo '```sh' - echo "gh release create v${VERSION} --title v${VERSION} --verify-tag --latest --generate-notes" - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + # NOTE: the GitHub Release is deliberately NOT cut here either. Marketplace + # listings are cut from a published Release, so creating one announces a + # version to consumers -- which must not happen before the wheel is on PyPI. + # It moved to the `promote` job at the bottom of this file, alongside the + # major-tag move, for the same reason. # Build + push the agent image HERE, in the same job that produced the # version, so the `:` tag is built from the correct pyproject (bumped @@ -450,3 +353,146 @@ jobs: # Trusted Publisher is configured on pypi.org for this repo + # workflow (release.yml) + environment (pypi); no password needed. packages-dir: dist/ + + # Everything CONSUMER-VISIBLE happens here, and only after the wheel is actually + # on PyPI: the moving major tag (`v0`, what every consumer pins) and the GitHub + # Release (what the Marketplace listing is cut from). + # + # WHY A SEPARATE JOB. The composite action installs `coder-eval==`, and the release commit bumps that pin. So moving `v0` before + # the wheel exists points every `uses: UiPath/coder_eval@v0` consumer at a pin + # that cannot resolve -- `uv tool install` 404s and their pipeline breaks. That was + # reachable two ways while both steps lived in the `release` job: publish-pypi is a + # separate `needs: release` job that can fail or sit waiting on the `pypi` + # environment gate, AND the tag move sat *before* "Build wheel + sdist", so a build + # failure stranded the pin without PyPI being involved at all. Ordering the tag move + # after the publish removes both, rather than detecting them after the fact. + # + # RE-RUNNABILITY IS THE POINT. The `release` job is NOT re-runnable -- re-running it + # would bump and tag a second version. This job is: the tag move is force-push + # idempotent and the Release create is existence-guarded. So a failure here (or in + # publish-pypi) is recovered by re-running the failed jobs from the Actions tab, + # with `v0` still pointing at the last fully-published release the whole time. That + # is why these steps can now fail LOUDLY instead of being swallowed by + # `continue-on-error` -- the previous best-effort + annotation dance existed only + # because a failure would have skipped publish-pypi and stranded the tag. + # + # RESIDUAL, ACCEPTED: the exact-version tag `vX.Y.Z` and `main` are pushed by the + # `release` job, so if publish-pypi fails they briefly reference an unpublished + # version. Narrower than the `v0` window by design -- `@v0` is the documented pin + # (see action.yml's header) and `@vX.Y.Z`/`@main` are opt-in -- and cleared by + # re-running publish-pypi. Closing it entirely would mean publishing to PyPI before + # pushing any git ref, which requires carrying the bumped commit + tag between jobs + # as an artifact; not worth the new failure modes. + promote: + name: Promote major tag and cut GitHub Release + needs: [release, publish-pypi] + # Real releases only. `released_version` is empty on a prerelease dispatch, which + # must never move the major tag or cut a Release. (A prerelease also skips + # publish-pypi's sibling path in spirit -- but note a *skipped* `needs` job blocks + # this one anyway, so this guard is belt-and-suspenders.) + if: needs.release.outputs.released_version != '' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + # Pushing the major tag needs the release app's credentials, same as the + # version-tag push in the `release` job: the workflow's GITHUB_TOKEN is + # contents: read, and tag writes are the app's job. + - name: Mint release app token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + # The version is interpolated into `ref:` below, so pin its shape first. It + # comes from `semantic-release version --print` (internal, not user input), so + # this is defence-in-depth against a malformed value producing a surprising ref + # rather than a mitigation for untrusted input. + - name: Validate version shape + env: + VERSION: ${{ needs.release.outputs.released_version }} + run: | + set -euo pipefail + [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "::error::refusing to promote a malformed version: '$VERSION'"; exit 1; } + echo "promoting v$VERSION" + + # Check out the released TAG, not main: main may have advanced since the + # release job ran, and the CHANGELOG slice below must be the one that shipped + # with this version. + - name: Checkout released tag + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: refs/tags/v${{ needs.release.outputs.released_version }} + fetch-depth: 0 # need tag objects to re-point the major tag + token: ${{ steps.app-token.outputs.token }} + + # The `v0` promotion itself. Force-move is idempotent, so re-running this job + # is always safe; force on a missing tag creates it (first release of a major). + - name: Move major action tag (vN -> this release) + env: + VERSION: ${{ needs.release.outputs.released_version }} + run: | + set -euo pipefail + MAJOR="v${VERSION%%.*}" + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + git tag -f "$MAJOR" "v${VERSION}" + git push -f origin "$MAJOR" + echo "Moved $MAJOR -> v${VERSION} (coder-eval==${VERSION} is on PyPI)" + + # semantic-release runs with --no-vcs-release (it also runs --no-push, and the + # commit is amended + the tag re-pointed afterwards), so it cannot create the + # Release itself -- it happens here, once the tag is on the remote AND the wheel + # is published. A published Release is what GitHub Marketplace listings are cut + # from, so every release needs one. (`gh release create` cannot tick the + # "Publish this Action to the Marketplace" checkbox -- that stays a one-time + # manual step in the GitHub UI on the first Release; every subsequent release + # then lists automatically.) + # + # Notes are the CHANGELOG section semantic-release generated for this version, + # sliced by .github/scripts/release_notes.py (a real module, so the regex is + # unit-tested -- see tests/test_release_notes.py); an empty result falls back to + # GitHub's generated notes. + # + # ACCEPTED RISK: those notes render commit subjects, i.e. squashed PR titles. + # The Release body is a first-party surface that GitHub also fans out in + # notification emails, so it carries text that was reviewed as *code*, not as + # markdown -- a PR title can land an arbitrary link in it. Bounded to + # content/link spoofing (GitHub strips raw HTML from release bodies) and gated + # by this repo's mandatory PR review. Revisit with `--draft` plus a human + # glance, or link-stripping in release_notes.py, if the repo ever takes drive-by + # contributions. + - name: Publish GitHub Release + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + # Passed via env (not interpolated into the script) per GitHub's + # injection guidance. + VERSION: ${{ needs.release.outputs.released_version }} + run: | + set -euo pipefail + # Existence-guarded so re-running this job after a partial failure is a + # no-op rather than a "release already exists" error. + if gh release view "v${VERSION}" >/dev/null 2>&1; then + echo "GitHub Release v${VERSION} already exists — nothing to do." + exit 0 + fi + # Written under RUNNER_TEMP, never the repo root: hatchling's default sdist + # file selection sweeps in untracked files at the root (verified -- it ships + # even git-ignored paths). The sdist is built in the `release` job, not here, + # but keeping the convention avoids re-learning it if that ever changes. + NOTES_FILE="${RUNNER_TEMP}/release-notes.md" + python3 .github/scripts/release_notes.py "$VERSION" "$NOTES_FILE" + # Empty notes file => no CHANGELOG section was found (the script already + # emitted the ::warning::); let GitHub generate the body instead. + if [ -s "$NOTES_FILE" ]; then + NOTES=(--notes-file "$NOTES_FILE") + else + NOTES=(--generate-notes) + fi + gh release create "v${VERSION}" \ + --title "v${VERSION}" \ + --verify-tag \ + --latest \ + "${NOTES[@]}" diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml new file mode 100644 index 00000000..1b101d74 --- /dev/null +++ b/.github/workflows/verify-published-action.yml @@ -0,0 +1,305 @@ +name: Verify Published Action + +# Verifies the PUBLISHED composite of (moving major tag + action.yml pin + PyPI +# wheel + Marketplace listing) actually resolves and runs. +# +# WHY THIS EXISTS SEPARATELY FROM pr-checks.yml. The `action-dogfood` job there +# runs `uses: ./` with `version: local`, which proves the action's code in a PR +# works. It never touches `v0` or PyPI, so it says nothing about the published +# artifact. This workflow is the other half. +# +# WHY THIS IS NOT A PR GATE. The published artifact does not change when someone +# opens a PR, so a PR-time job pulling `@v0` would re-test the PREVIOUS release on +# every PR -- paying agent tokens each time and going red for reasons the PR author +# cannot fix. That is how required checks get ignored. PR-time coverage stays as-is. + +on: + workflow_run: + workflows: ["Release"] + # Deliberately NOT filtered on `conclusion == 'success'`. The failure this + # workflow exists to catch -- publish-pypi failing after the release job pushed + # tags -- makes the Release run's conclusion `failure`. Gating on success would + # skip the check precisely when it matters most. + types: [completed] + # Catches drift a release cannot: a PyPI yank, the pinned setup-uv SHA, runner + # image changes, the @anthropic-ai/claude-code npm package, model deprecation, or + # the Marketplace listing being renamed/delisted. + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Serialize rather than cancel: a release-triggered run must not be killed by an + # overlapping nightly. + group: verify-published-action + cancel-in-progress: false + +jobs: + # TIER 1 -- free. No API spend, fully deterministic. This tier alone catches the + # tag/pin/PyPI desync, so it gates the paid tier below. + preflight: + name: Preflight (tag/pin/PyPI/Marketplace parity) + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout (full history for tags) + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + # Resolve the published triple from git alone: newest release tag, the moving + # major tag, and the `version:` pin baked into action.yml AT THAT TAG (not the + # working tree -- the tree is whatever main happens to be, which is not what + # consumers resolve). + - name: Check tag / pin parity + id: parity + run: | + set -euo pipefail + + # Release tags are strictly vX.Y.Z; prereleases are never tagged. + NEWEST=$(git tag -l 'v*' --sort=-v:refname \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1) + if [ -z "$NEWEST" ]; then echo "::error::no vX.Y.Z release tag found"; exit 1; fi + VERSION="${NEWEST#v}" + MAJOR="v${VERSION%%.*}" + echo "newest release tag: $NEWEST" + echo "moving major tag: $MAJOR" + + if ! git rev-parse -q --verify "refs/tags/$MAJOR" >/dev/null; then + echo "::error::moving major tag $MAJOR does not exist"; exit 1 + fi + + # The e2e job below hardcodes `uses: UiPath/coder_eval@v0` -- GitHub Actions + # forbids expressions in `uses:`, so it cannot be derived. Assert the major + # here so a 1.0.0 release fails loudly instead of silently leaving the paid + # tier testing a stale major forever. + if [ "$MAJOR" != "v0" ]; then + echo "::error::major tag is now $MAJOR, but the e2e job below pins @v0. Bump the \`uses:\` in this workflow." + exit 1 + fi + + # 1. The major tag must point at the newest release. + MAJOR_SHA=$(git rev-parse "refs/tags/$MAJOR^{commit}") + NEWEST_SHA=$(git rev-parse "refs/tags/$NEWEST^{commit}") + if [ "$MAJOR_SHA" != "$NEWEST_SHA" ]; then + echo "::error::$MAJOR points at $MAJOR_SHA but $NEWEST is $NEWEST_SHA -- consumers on @$MAJOR are not getting the newest release" + exit 1 + fi + + # 2. action.yml AT the major tag must pin the newest released version. + # Anchor mirrors release.yml's sed and tests/test_action_version_pin.py. + PIN=$(git show "$MAJOR:action.yml" \ + | sed -nE 's/^[[:space:]]*default: "([0-9]+\.[0-9]+\.[0-9]+)"[[:space:]]+# <-- kept in sync.*/\1/p') + if [ -z "$PIN" ]; then + echo "::error::could not read the version pin from action.yml at $MAJOR (the '# <-- kept in sync' anchor may have been detached)" + exit 1 + fi + echo "action.yml pin at $MAJOR: $PIN" + if [ "$PIN" != "$VERSION" ]; then + echo "::error::action.yml at $MAJOR pins coder-eval==$PIN but the newest release is $VERSION -- @$MAJOR consumers install the wrong version" + exit 1 + fi + + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + echo "parity OK: @$MAJOR -> $NEWEST -> coder-eval==$VERSION" + + # The stranded-pin check: does the version @v0 promises actually exist on PyPI? + # Retried because a release-triggered run can land seconds after the upload, + # before the index/CDN has propagated -- without the wait this would flake red + # on every release, which is how a check earns being ignored. + - name: Verify version exists on PyPI + env: + VERSION: ${{ steps.parity.outputs.version }} + run: | + set -euo pipefail + URL="https://pypi.org/pypi/coder-eval/${VERSION}/json" + for attempt in 1 2 3 4 5 6; do + CODE=$(curl -sS -o /dev/null -w '%{http_code}' "$URL" || echo 000) + if [ "$CODE" = "200" ]; then + echo "coder-eval==${VERSION} is on PyPI" + exit 0 + fi + echo "attempt $attempt: PyPI returned $CODE for ${VERSION}; waiting for propagation..." + sleep 20 + done + echo "::error title=Stranded action.yml pin::coder-eval==${VERSION} is NOT on PyPI, but the moving major tag points at an action.yml that installs it. Every 'uses: UiPath/coder_eval@v0' consumer fails at install. Re-run the Release workflow's publish-pypi job." + exit 1 + + # Catches the listing being renamed or delisted. The slug is derived from + # action.yml's `name:` rather than hardcoded -- and note GitHub does NOT + # convert underscores to hyphens: `coder_eval` is the live slug (verified; + # `coder-eval` 404s). tests/lint/action_docs.py (CE026) keeps the docs links + # consistent with the same `name:`. + - name: Verify Marketplace listing resolves + env: + # The version tag, which the parity step above proved is the same commit + # the major tag points at. + TAG_REF: v${{ steps.parity.outputs.version }} + run: | + set -euo pipefail + NAME=$(git show "${TAG_REF}:action.yml" \ + | sed -nE 's/^name:[[:space:]]*(.+)$/\1/p' | head -1 \ + | sed -E 's/^["'"'"']//; s/["'"'"']$//') + if [ -z "$NAME" ]; then echo "::error::could not read \`name:\` from action.yml"; exit 1; fi + SLUG=$(echo "$NAME" | tr '[:upper:]' '[:lower:]' | tr ' ' '-') + URL="https://github.com/marketplace/actions/${SLUG}" + echo "listing name: $NAME -> $URL" + + CODE=000 + for attempt in 1 2 3; do + CODE=$(curl -sSL -o /dev/null -w '%{http_code}' "$URL" || echo 000) + [ "$CODE" = "200" ] && break + echo "attempt $attempt: $CODE" + sleep 10 + done + + if [ "$CODE" = "200" ]; then + echo "Marketplace listing resolves." + elif [ "$CODE" = "429" ] || [ "$CODE" -ge 500 ] 2>/dev/null; then + # Transient / bot-throttled: warn, do not fail. Distinguishing this from a + # 404 is what keeps the check from crying wolf. + echo "::warning title=Marketplace check inconclusive::GitHub returned $CODE for $URL (rate-limited or transient), not treating as delisted." + else + echo "::error title=Marketplace listing missing::$URL returned $CODE. The listing may have been renamed or delisted, or action.yml's \`name:\` changed without the listing following." + exit 1 + fi + + - name: Install uv + uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 + + # Proves the wheel is installable and the console script works -- the exact two + # things the composite action does before it runs anything. + - name: Install from PyPI and smoke the CLI + env: + VERSION: ${{ steps.parity.outputs.version }} + run: | + set -euo pipefail + uv tool install "coder-eval==${VERSION}" + coder-eval --help > /dev/null + echo "coder-eval==${VERSION} installs and runs." + + # TIER 2 -- costs cents. Consumes the action exactly as a stranger would: + # `uses: UiPath/coder_eval@v0` with the default `version:` (never `local`), no repo + # checkout, and a task YAML written inline rather than one from this repo. That + # makes it a real consumer simulation instead of a self-referential run, and it + # doubles as a live proof that the documented agent-runtime prerequisite steps + # still work. + e2e: + name: End-to-end (published action, real API) + needs: preflight + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # No actions/checkout on purpose: a consumer's workspace does not contain this + # repo. Everything the run needs is written below or installed by the action. + # (The default experiment resolves from packaged resources, not the repo.) + + # The prerequisite steps the README/docs tell consumers to add. The action is + # agent-agnostic and installs no agent runtime. + - name: Set up Node.js 20 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "20" + - name: Install Claude CLI + run: npm install -g @anthropic-ai/claude-code + + - name: Write a consumer task YAML + run: | + set -euo pipefail + mkdir -p tasks + cat > tasks/published_smoke.yaml <<'YAML' + task_id: "published_action_smoke" + description: "Minimal task proving the published action installs and drives an agent." + initial_prompt: "Create a file named hello.txt in the current working directory containing exactly the single line: hello from the published action" + tags: [smoke] + + agent: + type: "claude-code" + permission_mode: "acceptEdits" + allowed_tools: ["Read", "Write", "Bash"] + # No host CLAUDE.md/settings to inherit here, but pinned explicitly so the + # run is isolated and cheap regardless of the runner's state. + setting_sources: [] + + success_criteria: + - type: "file_exists" + path: "hello.txt" + description: "The agent must create hello.txt." + YAML + echo "--- task YAML:"; cat tasks/published_smoke.yaml + + # continue-on-error, because this step's exit code is NOT the gate. The action + # exits with coder-eval's own code (action.yml combines them), and coder-eval + # exits 1 on any failed task -- so a model flake failing `file_exists` would + # redden this workflow even with minimum-task-score at 0.0, which does not + # neutralize that path. This check must answer "does the published action still + # work", not "is the model still good": the verification step below gates on + # ARTIFACTS instead. A genuine model/credential outage still surfaces there, via + # the zero-token assertion. + - name: Run the published action + id: run + continue-on-error: true + uses: UiPath/coder_eval@v0 # major asserted by the preflight job above + with: + # `version:` intentionally omitted -- the whole point is to exercise the + # default pin baked into action.yml at the v0 tag. + tasks: tasks/published_smoke.yaml + model: claude-haiku-4-5-20251001 + run-dir: runs/verify-published + junit-path: runs/verify-published/junit.xml + minimum-task-score: "0.0" + env: | + ANTHROPIC_API_KEY=${{ secrets.ANTHROPIC_API_KEY }} + + # THE ACTUAL GATE: the action's mechanics, not the model's answer. + - name: Verify action mechanics + env: + JUNIT: ${{ steps.run.outputs.junit-path }} + RUNDIR: ${{ steps.run.outputs.run-dir }} + STEP_OUTCOME: ${{ steps.run.outcome }} + run: | + set -euo pipefail + echo "action step outcome: $STEP_OUTCOME (informational -- artifacts are the gate)" + + # 1. Outputs are wired. + [ -n "$JUNIT" ] || { echo "::error::action did not set the junit-path output"; exit 1; } + [ -n "$RUNDIR" ] || { echo "::error::action did not set the run-dir output"; exit 1; } + + # 2. The JUnit report exists and is well-formed (trusted, self-generated + # input; our writer emits no DTDs/entities, so stdlib ET is fine). + [ -f "$JUNIT" ] || { echo "::error::JUnit report missing at $JUNIT"; exit 1; } + python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" + + # 3. run.json -- the consumer contract -- exists and describes a real run + # that reached the model. Non-zero tokens prove the credential passthrough + # and the agent runtime actually worked, WITHOUT asserting output quality. + [ -f "$RUNDIR/run.json" ] || { echo "::error::run.json missing in $RUNDIR"; exit 1; } + RUN_JSON="$RUNDIR/run.json" python3 <<'PY' + import json, os, sys + + data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) + rows = data.get("task_results") or [] + if not rows: + print("::error::run.json contains no task_results -- the action produced no run") + sys.exit(1) + tokens = sum(r.get("total_tokens") or 0 for r in rows) + for r in rows: + print(f" {r.get('task_id')}: status={r.get('final_status')} " + f"score={r.get('weighted_score')} tokens={r.get('total_tokens')}") + if tokens <= 0: + print("::error::no tokens consumed across any task -- the agent never reached the model " + "(credential passthrough, agent runtime, or backend wiring is broken)") + sys.exit(1) + print(f"published action mechanics OK: {len(rows)} task row(s), {tokens} tokens consumed") + PY + + - name: Upload run on failure + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: verify-published-runs + path: runs/verify-published/ + retention-days: 7 From b5a4b54bffb8220cbb9709da9a6daa62426d88a8 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 4 Aug 2026 14:59:39 -0700 Subject: [PATCH 2/9] fix(ci): code review fixes for the published-action verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings from a multi-model review (gemini-3.1-pro, gpt-5.6-sol), all verified by reproducing the failure before fixing: - `verify-published-action.yml`: `NEWEST=$(git tag -l … | grep … | head -1)` under `set -euo pipefail` aborts the step when grep matches nothing (exit 1) or head closes the pipe early (141), so the `if [ -z "$NEWEST" ]` diagnostic below it was dead code — a repo with no release tags got a bare exit 1 with no message. Reproduced both ways; `|| true` lets the emptiness check own every failure mode. - `verify-published-action.yml`: the Marketplace probe treated `403` and `000` as proof of delisting. GitHub commonly serves 403 to unauthenticated page fetches from CI runners, and `000` is curl failing outright (DNS/network/TLS) — both are "we learned nothing", not "it's gone". They now warn alongside 429/5xx; only 4xx proper still hard-fails. This was the exact cry-wolf failure the step's own comment set out to avoid. - `verify-published-action.yml`: the e2e gate ignored the action step's exit code entirely, which also hid regressions in the action's OWN exit logic (e.g. a broken score gate reddening a run whose every task succeeded) — a genuine "published action is broken" signal. Now conditional: tolerate a red step when any task under-performed (model flake), require green when all reported SUCCESS. Verified it fires on the regression case and stays quiet on the flake case. Note the reviewer's proposed patch keyed on `final_status`, which does not exist in run.json — `eval_result_to_task_dict` writes `status`. Implemented against the real key and confirmed the suggested form would have been dead on arrival. The same typo was live in this workflow's own diagnostic line (printing `status=None` every run); fixed. - `release.yml`: `gh release view` also matches a DRAFT or prerelease, so promote could skip creation and report success while announcing nothing to the Marketplace. Now normalizes with `gh release edit --draft=false --prerelease=false --latest`, making the job's idempotency claim true in fact. Also records two deferred harness candidates: CE034 for the dead-guard shell pattern (confirmed NOT caught by actionlint+shellcheck, so the existing actionlint candidate does not subsume it), and runtime-key parity for the `run.json` keys that shell consumers depend on but no test binds. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 24 ++++++++++ .github/workflows/release.yml | 8 +++- .github/workflows/verify-published-action.yml | 44 ++++++++++++++++--- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 3957153f..658dbbbb 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -391,3 +391,27 @@ with the two `action.yml` items above — one considered change to the action's violations on `main` before this bug, one on this PR). Worth a real look next time `agents/` is touched, since a second agent adding its own disconnected sleep-loop constant would reintroduce the exact same shape. + +## From 2026-08-04 published-action verification review + +- [ ] **CE034 — `VAR=$(… | grep …)` under `set -e` followed by an emptiness check + is a dead diagnostic.** With `set -euo pipefail`, a pipeline whose `grep` matches + nothing exits 1, so the assignment aborts the step *before* the + `if [ -z "$VAR" ]; then echo "::error::…"` branch that was written to report it — + the operator gets a bare exit 1 with no message. Also applies to `head -1` + closing the pipe early (SIGPIPE 141). Fix is `|| true` on the substitution, + letting the emptiness check own every failure mode. Detectable by matching + `\w+=\$\(.*\|\s*(grep|head)\b` inside a `run:` body whose script sets `-e`, then + requiring `|| true`/`|| :` on the same logical line. Caught by a reviewer in + `verify-published-action.yml`; **`actionlint` + shellcheck do NOT flag it** + (verified against the exact snippet), so the actionlint candidate above does not + subsume this one. +- [ ] **Runtime-key parity for `run.json` consumers outside `src/`.** The e2e gate in + `verify-published-action.yml` reads `task_results[*].status` / `weighted_score` / + `total_tokens`, and `action.yml`'s score gate reads `weighted_score` / `task_id`. + These are string keys in shell/YAML that no test or type-checker binds to + `eval_result_to_task_dict` (`reports_experiment.py`), so renaming a key there + silently turns an external gate into a no-op — a reviewer here proposed + `final_status`, which does not exist in `run.json` and would have made a new + assertion dead on arrival. Guard: assert the key set that non-Python consumers + depend on, mirroring how CE030 pins doc/schema parity. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c49aba7c..e60f3622 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -473,9 +473,13 @@ jobs: run: | set -euo pipefail # Existence-guarded so re-running this job after a partial failure is a - # no-op rather than a "release already exists" error. + # no-op rather than a "release already exists" error. `gh release view` + # also matches a DRAFT or prerelease, which would announce nothing to the + # Marketplace -- so normalize rather than trusting mere existence, keeping + # this job idempotent in fact and not just in the happy case. if gh release view "v${VERSION}" >/dev/null 2>&1; then - echo "GitHub Release v${VERSION} already exists — nothing to do." + gh release edit "v${VERSION}" --draft=false --prerelease=false --latest + echo "GitHub Release v${VERSION} already existed — normalized to published/latest." exit 0 fi # Written under RUNNER_TEMP, never the repo root: hatchling's default sdist diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 1b101d74..cad32dd5 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -60,8 +60,13 @@ jobs: set -euo pipefail # Release tags are strictly vX.Y.Z; prereleases are never tagged. + # `|| true` is required, not defensive noise: under `pipefail` this pipeline + # exits non-zero when grep matches nothing (and, in theory, 141 if `head` + # closes the pipe early), so `set -e` would kill the step BEFORE the + # empty-check below — making its error message dead code. Any failure mode + # lands as an empty $NEWEST, which the check reports properly. NEWEST=$(git tag -l 'v*' --sort=-v:refname \ - | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1) + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1 || true) if [ -z "$NEWEST" ]; then echo "::error::no vX.Y.Z release tag found"; exit 1; fi VERSION="${NEWEST#v}" MAJOR="v${VERSION%%.*}" @@ -158,10 +163,15 @@ jobs: if [ "$CODE" = "200" ]; then echo "Marketplace listing resolves." - elif [ "$CODE" = "429" ] || [ "$CODE" -ge 500 ] 2>/dev/null; then - # Transient / bot-throttled: warn, do not fail. Distinguishing this from a - # 404 is what keeps the check from crying wolf. - echo "::warning title=Marketplace check inconclusive::GitHub returned $CODE for $URL (rate-limited or transient), not treating as delisted." + # Inconclusive, NOT proof of delisting — warn and move on. Only 404 (and + # other 4xx) is treated as a real signal. This split is what keeps the + # check from crying wolf: + # 000 -> curl itself failed (DNS/network/TLS); we learned nothing. + # 403 -> GitHub commonly serves this to unauthenticated/bot page fetches + # from CI runners; it means "not shown to you", not "not there". + # 429/5xx -> rate-limited or GitHub-side transient. + elif [ "$CODE" = "000" ] || [ "$CODE" = "403" ] || [ "$CODE" = "429" ] || [ "$CODE" -ge 500 ] 2>/dev/null; then + echo "::warning title=Marketplace check inconclusive::GitHub returned $CODE for $URL (network failure, throttling, or transient), not treating as delisted." else echo "::error title=Marketplace listing missing::$URL returned $CODE. The listing may have been renamed or delisted, or action.yml's \`name:\` changed without the listing following." exit 1 @@ -285,15 +295,35 @@ jobs: if not rows: print("::error::run.json contains no task_results -- the action produced no run") sys.exit(1) + # Key is "status" (eval_result_to_task_dict writes FinalStatus there); there is + # no "final_status" key in run.json rows, and reading one would silently + # evaluate to None and make the exit-contract assertion below dead code. + statuses = [r.get("status") for r in rows] tokens = sum(r.get("total_tokens") or 0 for r in rows) for r in rows: - print(f" {r.get('task_id')}: status={r.get('final_status')} " + print(f" {r.get('task_id')}: status={r.get('status')} " f"score={r.get('weighted_score')} tokens={r.get('total_tokens')}") + if tokens <= 0: print("::error::no tokens consumed across any task -- the agent never reached the model " "(credential passthrough, agent runtime, or backend wiring is broken)") sys.exit(1) - print(f"published action mechanics OK: {len(rows)} task row(s), {tokens} tokens consumed") + + # Exit-contract check, conditional on the model having actually performed. + # Ignoring the step's exit code entirely (see the continue-on-error rationale + # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a + # broken score gate failing the step even though every task succeeded. That is + # precisely a "published action is broken" signal and must be caught. So: + # tolerate a red step when any task under-performed (model flake, not our bug), + # but require green when all of them succeeded. + if all(s == "SUCCESS" for s in statuses) and os.environ.get("STEP_OUTCOME") != "success": + print("::error::every task reported SUCCESS but the published action step exited " + f"non-zero (outcome={os.environ.get('STEP_OUTCOME')!r}) -- the action's exit-code " + "contract or its score gate is broken for consumers") + sys.exit(1) + + print(f"published action mechanics OK: {len(rows)} task row(s), {tokens} tokens consumed, " + f"statuses={statuses}") PY - name: Upload run on failure From 9d078a5e739567053e2ec28dc5d477a9dfaaa3e0 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Tue, 4 Aug 2026 15:06:27 -0700 Subject: [PATCH 3/9] fix(ci): harden promote ordering and stop preflight misdiagnosing healthy lag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review pass (Opus) on top of the gemini/gpt-5 findings. Six issues, each reproduced before fixing: 1. `v0` force-move was idempotent but NOT monotonic. GitHub keeps "Re-run failed jobs" live for 30 days, so replaying an OLD release's promote (0.9.5 fails at publish-pypi, operator ships 0.9.6, later cleans up the red 0.9.5 run) walked `v0` BACKWARDS and silently downgraded every consumer. The removed comment claimed re-running "is always safe" — force-push is only self-idempotent and says nothing about ordering. Now refuses to promote anything but the newest release tag, with a message naming the version to promote instead. 2. preflight treated the holding state THIS PR introduces as a defect. Because promote now moves `v0` only after publish-pypi, `v0` legitimately lags for the whole interval — including publish-pypi failing (where @v0 consumers are perfectly HEALTHY on the previous release) and the `pypi` environment approval window. Old code hard-failed at "consumers are not getting the newest release" and never reached the accurate stranded-pin diagnostic; every nightly during an approval window would have gone red on a working artifact. The two halves of this PR contradicted each other. The hard gate is now the consumer contract — the version @v0's action.yml PINS must be installable — and lag is classified: newest on PyPI => promote didn't run (hard fail, actionable); newest absent => release merely incomplete (warning, consumers unaffected). 3. `publish-pypi` was not re-runnable, which the whole recovery story assumes. An upload that succeeds but whose step then fails (lost response, timeout) gets 400 "File already exists" forever, so promote could never run for a version that IS published. Added `skip-existing: true`. 4. `|| echo 000` double-appended: curl's own `-w '%{http_code}'` already prints 000 on transport failure, so CODE became the literal "000000" and matched neither the transient allowlist nor 5xx. A DNS/TLS blip was reported as "renamed or delisted" / a stranded pin. Verified `000000` empirically; the previous commit's attempt to allowlist "000" was therefore ineffective. Removed the append in both probes and split "unreachable" from "absent" in the messages. 5. e2e gate was load-bearing on composite `outputs:` surviving a continue-on-error failure — undocumented behavior, and if it does not hold every model flake reddens the workflow with "did not set the junit-path output", defeating the artifact-gate design. File checks now use the literal paths the workflow itself passes in `with:`; output wiring is asserted separately, hard only when the step went green (where propagation is guaranteed) and as a warning otherwise. 6. promote's `if:` failed in the SKIP direction. Gated on `needs.release.outputs.released_version != ''`, a lost output on a partial re-run resolves to skipped-green: green re-run, tag never moved, no Release. Now discriminates prereleases on `github.ref` (the same signal "Determine release mode" uses, and one that cannot evaporate), with emptiness enforced inside the job so a lost output is RED, not silent. Also fixed two Low findings while here: removed dead `git config user.email/name` (a lightweight `git tag -f` needs no committer identity), and scoped the paid e2e tier off branch-dispatched prereleases, which cannot change the published artifact. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release.yml | 60 +++++-- .github/workflows/verify-published-action.yml | 164 ++++++++++++++---- 2 files changed, 173 insertions(+), 51 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e60f3622..c1096c89 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -353,6 +353,13 @@ jobs: # Trusted Publisher is configured on pypi.org for this repo + # workflow (release.yml) + environment (pypi); no password needed. packages-dir: dist/ + # Required for the `promote` job's recovery story to actually work. Without + # it, a step that fails AFTER a successful upload (lost response, job + # timeout) can never be re-run: PyPI answers 400 "File already exists", this + # job stays permanently red, so `promote` can never run and the major tag is + # never moved for a version that IS published -- the stranded state from the + # other direction. Trusted-Publishing scoping is unaffected. + skip-existing: true # Everything CONSUMER-VISIBLE happens here, and only after the wheel is actually # on PyPI: the moving major tag (`v0`, what every consumer pins) and the GitHub @@ -387,11 +394,15 @@ jobs: promote: name: Promote major tag and cut GitHub Release needs: [release, publish-pypi] - # Real releases only. `released_version` is empty on a prerelease dispatch, which - # must never move the major tag or cut a Release. (A prerelease also skips - # publish-pypi's sibling path in spirit -- but note a *skipped* `needs` job blocks - # this one anyway, so this guard is belt-and-suspenders.) - if: needs.release.outputs.released_version != '' + # Real releases only, discriminated on the DISPATCHED REF rather than on a `needs` + # output. Prerelease mode is defined by the ref (see "Determine release mode"), so + # this is the same signal, and it cannot silently evaporate: were this gated on + # `needs.release.outputs.released_version != ''` and that output failed to carry + # over into a partial "Re-run failed jobs" attempt, the job would resolve to + # SKIPPED-GREEN -- the operator sees a green re-run while the major tag never moves + # and no Release is cut. Emptiness is enforced inside the job instead, by + # "Validate version shape", so a lost output is a RED job, not a silent no-op. + if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -405,15 +416,22 @@ jobs: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} - # The version is interpolated into `ref:` below, so pin its shape first. It - # comes from `semantic-release version --print` (internal, not user input), so - # this is defence-in-depth against a malformed value producing a surprising ref - # rather than a mitigation for untrusted input. + # Two jobs in one: (1) the version is interpolated into `ref:` below, so pin its + # shape first -- defence-in-depth against a malformed value producing a surprising + # ref (the value is `semantic-release version --print` output, first-party, not + # untrusted input); (2) this is the ENFORCEMENT POINT for a missing version, which + # the job's `if:` deliberately no longer gates on. An empty string fails the regex, + # so a `needs` output lost across a partial re-run surfaces as a red job with a + # clear message instead of a silently skipped promotion. - name: Validate version shape env: VERSION: ${{ needs.release.outputs.released_version }} run: | set -euo pipefail + if [ -z "$VERSION" ]; then + echo "::error title=Release version unavailable::needs.release.outputs.released_version is empty. On a real release it is always set, so this most likely means the output did not carry over into a partial re-run -- re-run the whole Release workflow's remaining jobs, or promote by hand." + exit 1 + fi [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { echo "::error::refusing to promote a malformed version: '$VERSION'"; exit 1; } echo "promoting v$VERSION" @@ -428,16 +446,32 @@ jobs: fetch-depth: 0 # need tag objects to re-point the major tag token: ${{ steps.app-token.outputs.token }} - # The `v0` promotion itself. Force-move is idempotent, so re-running this job - # is always safe; force on a missing tag creates it (first release of a major). + # The `v0` promotion itself. Force-move makes re-running THIS run's job safe, but + # force-push is only self-idempotent -- it says nothing about ORDERING. GitHub + # keeps "Re-run failed jobs" available for 30 days, so replaying an OLD release's + # promote (e.g. 0.9.5 failed at publish-pypi, the operator moved on and shipped + # 0.9.6, then later cleaned up the red 0.9.5 run) would walk `v0` BACKWARDS and + # silently downgrade every consumer. The monotonicity guard below is what makes + # "re-running is safe" actually true. No `-a`/`-m`, so this is a lightweight tag: + # a plain ref write needing no committer identity. - name: Move major action tag (vN -> this release) env: VERSION: ${{ needs.release.outputs.released_version }} run: | set -euo pipefail MAJOR="v${VERSION%%.*}" - git config user.email "github-actions[bot]@users.noreply.github.com" - git config user.name "github-actions[bot]" + + # Refuse to promote anything but the newest release tag. + NEWEST=$(git tag -l 'v*' --sort=-v:refname \ + | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | head -1 || true) + if [ -z "$NEWEST" ]; then + echo "::error::no vX.Y.Z release tag found; refusing to move $MAJOR"; exit 1 + fi + if [ "$NEWEST" != "v${VERSION}" ]; then + echo "::error title=Refusing to move $MAJOR backwards::this run promotes v${VERSION}, but ${NEWEST} is the newest release tag. Moving $MAJOR would downgrade every 'uses: UiPath/coder_eval@$MAJOR' consumer. If you are recovering an old release, promote ${NEWEST} instead." + exit 1 + fi + git tag -f "$MAJOR" "v${VERSION}" git push -f origin "$MAJOR" echo "Moved $MAJOR -> v${VERSION} (coder-eval==${VERSION} is on PyPI)" diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index cad32dd5..0b784b78 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -86,16 +86,9 @@ jobs: exit 1 fi - # 1. The major tag must point at the newest release. - MAJOR_SHA=$(git rev-parse "refs/tags/$MAJOR^{commit}") - NEWEST_SHA=$(git rev-parse "refs/tags/$NEWEST^{commit}") - if [ "$MAJOR_SHA" != "$NEWEST_SHA" ]; then - echo "::error::$MAJOR points at $MAJOR_SHA but $NEWEST is $NEWEST_SHA -- consumers on @$MAJOR are not getting the newest release" - exit 1 - fi - - # 2. action.yml AT the major tag must pin the newest released version. - # Anchor mirrors release.yml's sed and tests/test_action_version_pin.py. + # The pin baked into action.yml AT the major tag -- what @v0 consumers + # actually install. Anchor mirrors release.yml's sed and + # tests/test_action_version_pin.py. PIN=$(git show "$MAJOR:action.yml" \ | sed -nE 's/^[[:space:]]*default: "([0-9]+\.[0-9]+\.[0-9]+)"[[:space:]]+# <-- kept in sync.*/\1/p') if [ -z "$PIN" ]; then @@ -103,36 +96,100 @@ jobs: exit 1 fi echo "action.yml pin at $MAJOR: $PIN" - if [ "$PIN" != "$VERSION" ]; then - echo "::error::action.yml at $MAJOR pins coder-eval==$PIN but the newest release is $VERSION -- @$MAJOR consumers install the wrong version" - exit 1 - fi - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "parity OK: @$MAJOR -> $NEWEST -> coder-eval==$VERSION" + # Does the major tag lag the newest release tag? Since release.yml's + # `promote` job moves $MAJOR only AFTER publish-pypi succeeds, lag is a + # LEGITIMATE transient state (publish-pypi still awaiting the `pypi` + # environment approval, or having failed) -- NOT automatically a defect, and + # in the failed case @v0 consumers are perfectly healthy on the previous + # release. So this only classifies; the "Classify major-tag lag" step below + # decides, once it knows whether the newest version reached PyPI. Failing + # here would redden a healthy published artifact and pre-empt the accurate + # diagnostic, which is how a check earns being ignored. + MAJOR_SHA=$(git rev-parse "refs/tags/$MAJOR^{commit}") + NEWEST_SHA=$(git rev-parse "refs/tags/$NEWEST^{commit}") + if [ "$MAJOR_SHA" = "$NEWEST_SHA" ]; then + LAGGING=false + # Same commit, so the pin there MUST be the newest version. A mismatch means + # the release's action.yml sed/amend silently didn't take -- always a bug. + if [ "$PIN" != "$VERSION" ]; then + echo "::error::action.yml at $MAJOR pins coder-eval==$PIN but that same commit is release $VERSION -- release.yml's pin bump did not take, so @$MAJOR consumers install the wrong version" + exit 1 + fi + else + LAGGING=true + echo "note: $MAJOR ($MAJOR_SHA) lags newest release $NEWEST ($NEWEST_SHA) -- classifying below" + fi - # The stranded-pin check: does the version @v0 promises actually exist on PyPI? - # Retried because a release-triggered run can land seconds after the upload, - # before the index/CDN has propagated -- without the wait this would flake red - # on every release, which is how a check earns being ignored. - - name: Verify version exists on PyPI + { + echo "pin=$PIN" + echo "newest=$VERSION" + echo "lagging=$LAGGING" + } >> "$GITHUB_OUTPUT" + echo "parity OK: @$MAJOR promises coder-eval==$PIN (newest release $VERSION, lagging=$LAGGING)" + + # THE CONSUMER CONTRACT, and the hard gate of this tier: whatever version @v0's + # action.yml promises must be installable. This is what breaks a stranger's + # pipeline when it is false, so it is checked against the PIN rather than the + # newest tag. Retried because a release-triggered run can land seconds after the + # upload, before the index/CDN has propagated -- without the wait this would + # flake red on every release, which is how a check earns being ignored. + - name: Verify @v0's pinned version is installable from PyPI env: - VERSION: ${{ steps.parity.outputs.version }} + PIN: ${{ steps.parity.outputs.pin }} run: | set -euo pipefail - URL="https://pypi.org/pypi/coder-eval/${VERSION}/json" + URL="https://pypi.org/pypi/coder-eval/${PIN}/json" + CODE=000 for attempt in 1 2 3 4 5 6; do - CODE=$(curl -sS -o /dev/null -w '%{http_code}' "$URL" || echo 000) + # No `|| echo 000`: curl's own `-w '%{http_code}'` already prints 000 on a + # transport failure, so appending another would yield the literal "000000" + # and defeat every comparison below. `|| true` only absorbs curl's non-zero + # exit under `set -e`. + CODE=$(curl -sS -o /dev/null -w '%{http_code}' "$URL" || true) if [ "$CODE" = "200" ]; then - echo "coder-eval==${VERSION} is on PyPI" + echo "coder-eval==${PIN} is on PyPI -- @v0 consumers can install." exit 0 fi - echo "attempt $attempt: PyPI returned $CODE for ${VERSION}; waiting for propagation..." + echo "attempt $attempt: PyPI returned $CODE for ${PIN}; waiting for propagation..." sleep 20 done - echo "::error title=Stranded action.yml pin::coder-eval==${VERSION} is NOT on PyPI, but the moving major tag points at an action.yml that installs it. Every 'uses: UiPath/coder_eval@v0' consumer fails at install. Re-run the Release workflow's publish-pypi job." + + # 000 means we never reached PyPI at all -- that is our problem, not a + # stranded pin, and must not be reported as one. + if [ "$CODE" = "000" ]; then + echo "::error title=PyPI unreachable::could not reach $URL after 6 attempts (curl transport failure). This check is inconclusive -- it does NOT mean coder-eval==${PIN} is missing." + exit 1 + fi + echo "::error title=Stranded action.yml pin::coder-eval==${PIN} is NOT on PyPI (HTTP $CODE), but @v0 points at an action.yml that installs it. Every 'uses: UiPath/coder_eval@v0' consumer fails at install. Re-run the Release workflow's publish-pypi job, then its promote job." exit 1 + # Only runs when the major tag lags. Distinguishes the two causes, which need + # opposite verdicts -- the whole reason the parity step above does not fail on lag. + - name: Classify major-tag lag + if: steps.parity.outputs.lagging == 'true' + env: + PIN: ${{ steps.parity.outputs.pin }} + NEWEST: ${{ steps.parity.outputs.newest }} + run: | + set -euo pipefail + CODE=$(curl -sS -o /dev/null -w '%{http_code}' "https://pypi.org/pypi/coder-eval/${NEWEST}/json" || true) + if [ "$CODE" = "200" ]; then + # The newest version IS published, so `promote` should have moved the tag + # and did not. Consumers are stuck an entire release behind: actionable. + echo "::error title=promote did not run::coder-eval==${NEWEST} is on PyPI but @v0 still promises ${PIN}. The Release workflow's promote job was skipped or failed -- re-run it to move the major tag." + exit 1 + fi + if [ "$CODE" = "000" ]; then + echo "::warning title=Lag classification inconclusive::could not reach PyPI to check whether ${NEWEST} was published; @v0 still promises ${PIN}, which the previous step verified is installable." + exit 0 + fi + # Newest is tagged but unpublished => publish-pypi never completed (failed, or + # still awaiting the `pypi` environment approval). @v0 consumers are HEALTHY on + # ${PIN}; the release is merely incomplete. The Release run is red for this + # already, so warn rather than duplicating a red on a working artifact. + echo "::warning title=Release incomplete::${NEWEST} is tagged but not on PyPI (HTTP $CODE), so @v0 correctly still promises ${PIN}. Consumers are unaffected. Finish the release by re-running publish-pypi (and promote), or this lag will persist." + # Catches the listing being renamed or delisted. The slug is derived from # action.yml's `name:` rather than hardcoded -- and note GitHub does NOT # convert underscores to hyphens: `coder_eval` is the live slug (verified; @@ -155,7 +212,10 @@ jobs: CODE=000 for attempt in 1 2 3; do - CODE=$(curl -sSL -o /dev/null -w '%{http_code}' "$URL" || echo 000) + # See the PyPI step: curl already prints 000 on transport failure, so + # `|| echo 000` would produce the literal "000000" and make the transient + # branch below unreachable for the commonest transient of all. + CODE=$(curl -sSL -o /dev/null -w '%{http_code}' "$URL" || true) [ "$CODE" = "200" ] && break echo "attempt $attempt: $CODE" sleep 10 @@ -200,6 +260,12 @@ jobs: e2e: name: End-to-end (published action, real API) needs: preflight + # Skip the paid tier for a Release run that cannot have changed the published + # artifact: a PRERELEASE dispatch comes from a non-default branch and by design + # never tags, never moves the major tag, and never cuts a Release. The free tier + # still runs. Schedule/dispatch events have no workflow_run context, so the first + # clause lets them through. + if: github.event_name != 'workflow_run' || github.event.workflow_run.head_branch == github.event.repository.default_branch runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -267,22 +333,44 @@ jobs: # THE ACTUAL GATE: the action's mechanics, not the model's answer. - name: Verify action mechanics env: - JUNIT: ${{ steps.run.outputs.junit-path }} - RUNDIR: ${{ steps.run.outputs.run-dir }} + # Asserted against the LITERAL paths passed in the `with:` block above, not + # against steps.run.outputs.*. The action step is continue-on-error, and + # whether a composite action's `outputs:` mapping still propagates when an + # embedded step exits non-zero is not a documented guarantee -- if it does + # not, keying the file checks off the outputs would turn every model flake + # into "action did not set the junit-path output" and defeat the whole + # artifact-gate design. Output wiring is still checked below, but separately + # and only where it is meaningful. + RUNDIR: runs/verify-published + JUNIT: runs/verify-published/junit.xml + OUT_JUNIT: ${{ steps.run.outputs.junit-path }} + OUT_RUNDIR: ${{ steps.run.outputs.run-dir }} STEP_OUTCOME: ${{ steps.run.outcome }} run: | set -euo pipefail - echo "action step outcome: $STEP_OUTCOME (informational -- artifacts are the gate)" + echo "action step outcome: $STEP_OUTCOME (not the gate by itself -- see below)" - # 1. Outputs are wired. - [ -n "$JUNIT" ] || { echo "::error::action did not set the junit-path output"; exit 1; } - [ -n "$RUNDIR" ] || { echo "::error::action did not set the run-dir output"; exit 1; } - - # 2. The JUnit report exists and is well-formed (trusted, self-generated - # input; our writer emits no DTDs/entities, so stdlib ET is fine). - [ -f "$JUNIT" ] || { echo "::error::JUnit report missing at $JUNIT"; exit 1; } + # 1. The JUnit report exists at the requested path and is well-formed + # (trusted, self-generated input; our writer emits no DTDs/entities, so + # stdlib ET is fine). + if [ ! -f "$JUNIT" ]; then + echo "::error::no JUnit report at $JUNIT. If the action failed during install, the pinned version is probably not installable -- check the preflight job's PyPI result." + exit 1 + fi python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" + # 1b. Output wiring, as its own assertion with its own message. Only hard-fail + # when the step went green, where propagation is guaranteed; otherwise the + # emptiness is ambiguous (broken action vs. runner not mapping outputs of + # a failed composite) and must not be reported as a broken contract. + if [ "$STEP_OUTCOME" = "success" ]; then + [ "$OUT_JUNIT" = "$JUNIT" ] || { echo "::error::action's junit-path output is '$OUT_JUNIT', expected '$JUNIT'"; exit 1; } + [ "$OUT_RUNDIR" = "$RUNDIR" ] || { echo "::error::action's run-dir output is '$OUT_RUNDIR', expected '$RUNDIR'"; exit 1; } + echo "outputs wired correctly." + elif [ -z "$OUT_JUNIT" ] || [ -z "$OUT_RUNDIR" ]; then + echo "::warning::action step was red and its outputs are empty; cannot tell whether the action failed to set them or the runner does not map outputs of a failed composite. Artifact checks below are authoritative." + fi + # 3. run.json -- the consumer contract -- exists and describes a real run # that reached the model. Non-zero tokens prove the credential passthrough # and the agent runtime actually worked, WITHOUT asserting output quality. From 751d1c2b4885ae63de9cfb46cd4ffb08def54bb6 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 6 Aug 2026 06:33:29 -0700 Subject: [PATCH 4/9] =?UTF-8?q?fix(ci):=20address=20PR=20#81=20review=20?= =?UTF-8?q?=E2=80=94=20undefined=20step=20output,=20dead=20job=20gate,=20C?= =?UTF-8?q?E035?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both blockers from the multi-model review, plus every non-blocking finding that held up on inspection. Blockers: * `steps.parity.outputs.version` does not exist (the step writes pin/newest/ lagging), so `TAG_REF` expanded to the bare `v`, `git show "v:action.yml"` exited 128 under `set -euo pipefail`, and preflight was red on 100% of runs — taking the paid e2e tier (`needs: preflight`) with it. Keyed each reader off a value that exists: a new `major` output for the Marketplace step, and `pin` for the install/smoke step (installing `newest` fails during a legitimate lagging state while @v0 consumers are healthy). * Removed publish-pypi's `if: needs.release.outputs.version != ''`. Dead ("Resolve published version" already exits 1 on empty) and dangerous: on a partial re-run it resolved to SKIPPED-green, which — since promote needs [release, publish-pypi] — also skipped the promotion, for a green run that published no wheel and never moved v0. Emptiness is now asserted in-job, as promote does. Resilience and diagnosis: * PyPI probes gain the 403/429/5xx-vs-404 split the Marketplace probe already performs, so a throttle no longer reports "Stranded action.yml pin" and sends the operator to re-publish a healthy version. * The zero-token gate branches on run.json's error_category: upstream categories warn (inconclusive), everything else stays a hard wiring error. * ERROR/BUILD_FAILED with a non-upstream category now fail rather than being tolerated as model flakes. * JUnit assertion is no longer vacuous (testcase count >= task_results rows), the output-wiring check warns on a present-but-wrong value, and the run-dir upload is `always()` so the tolerated-red case keeps its evidence. Supply chain and least privilege: * skip-existing made a green publish stop proving PyPI serves this run's wheel; a new step compares urls[].digests.sha256 against sha256sum dist/*. Mismatch is fatal, an unreadable index is a warning (propagation lag must not redden a successful publish). * permission-contents: write on both app-token mints; promote drops the inherited packages: write. Guardrails, so this class cannot ship again: * CE035 (tests/lint/workflow_outputs.py) resolves every steps./needs. outputs reference to a real writer; its negative test is the exact shape of the bug above. actionlint models steps.*.outputs as an open string map and does not catch it. * tests/test_verify_published_workflow.py binds the four couplings nothing asserted: the workflow_run display-name link to release.yml, Marketplace slug parity with the tested marketplace_slug() over a punctuation table, all three `# <-- kept in sync` anchor readers, and the inline consumer task YAML loading through the real load_task. Docs: CONTRIBUTING gains a release runbook (job table, recovery flows, the nightly's annotation taxonomy); the GHCR image's exemption from the promote ordering and the unpinned agent-runtime install are recorded as accepted risks. Deferred items (script extraction, score-gate failure direction, CE036/CE037/ CE040) are booked in .claude/harness-candidates.md. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 39 +++ .github/workflows/release.yml | 132 ++++++++- .github/workflows/verify-published-action.yml | 158 ++++++++--- CLAUDE.md | 2 +- CONTRIBUTING.md | 52 ++++ tests/lint/workflow_outputs.py | 250 ++++++++++++++++++ tests/test_custom_lint.py | 93 +++++++ tests/test_verify_published_workflow.py | 217 +++++++++++++++ 8 files changed, 906 insertions(+), 37 deletions(-) create mode 100644 tests/lint/workflow_outputs.py create mode 100644 tests/test_verify_published_workflow.py diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 658dbbbb..c19a121a 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -406,6 +406,45 @@ with the two `action.yml` items above — one considered change to the action's `verify-published-action.yml`; **`actionlint` + shellcheck do NOT flag it** (verified against the exact snippet), so the actionlint candidate above does not subsume this one. +- [ ] **CE036 — ban the skipped-green job gate.** Fail a job-level `if:` in + `.github/workflows/**` whose only discriminator is an emptiness/equality test on + `needs..outputs.`. A lost output on a partial "Re-run failed jobs" resolves + the job to SKIPPED-**green**, so an operator sees a green re-run while nothing ran. + Fixed by hand twice now: `promote` was designed around the hazard, and + `publish-pypi`'s `if: needs.release.outputs.version != ''` (dead *and* dangerous — a + skipped publish also skipped `promote`) was removed in the follow-up review. CE035 + catches the *typo* class; this catches the *shape*. Escape hatch: inline + `# noqa: CE036 — ` for value-driven gates that cannot strand a release. +- [ ] **CE037 — `if: failure()` is wrong in a job containing a `continue-on-error` + step.** Require `always()` (or a reference to the tolerated step's + `steps..outcome`) on diagnostic/upload steps in such a job. Fixed by hand in + `verify-published-action.yml`: the run dir was discarded in exactly the tolerated-red + case the gate is designed around, because a tolerated red leaves the job green and + `failure()` never fires. Pure YAML shape check, ~30 lines. +- [ ] **CE040 — cap inline `run:` bodies; oversized decision logic belongs in + `.github/scripts/`.** `verify-published-action.yml`'s parity step (~70 lines, 7 + decision points) and its e2e gate (~66 lines, switching from bash to a `python3` + heredoc mid-step) are 10-20-branch units invisible to `make check`, `make lint`, + `pyright` and coverage — which is the structural reason the `steps.parity.outputs.version` + bug survived to `main`. Analogous to CE022's statement cap; composes with CE032/CE033. + Deferred as a refactor, not a fix: extraction touches all 423 lines of a workflow that + cannot be exercised before merge, and CE035 + `tests/test_verify_published_workflow.py` + now cover the specific failure classes. Precedent for the extraction: + `.github/scripts/release_notes.py` + `tests/test_release_notes.py`. +- [ ] **Exercise the Action's score gate in the FAILING direction.** Both + consumer-simulating jobs pass `minimum-task-score: "0.0"` + (`verify-published-action.yml`'s `e2e`, `pr-checks.yml`'s `action-dogfood`), so the gate + is only ever proven to *pass*. The new exit-contract assertion catches a gate that + wrongly fails; nothing catches one that wrongly passes — the direction that silently + disables every consumer's quality gate. Needs a second invocation with an unmeetable + score floor, i.e. a second paid agent run per nightly; deferred on cost, and better + placed in `action-dogfood` (PR-time, already paying) than in the cron. +- [ ] **Extend CE026's `REQUIRED_PREREQ_TOKENS` anchor to the `e2e` job.** The lint pins + the documented Node + `@anthropic-ai/claude-code` prerequisite steps to a single + executable reference (`action-dogfood` in `pr-checks.yml`, via + `tests/lint/action_docs.py::DOGFOOD_JOB`). `verify-published-action.yml`'s `e2e` job is + now a third copy of the same two steps — and the truer consumer proof (no checkout, + published action, default pin) — so the two can drift while the docs follow only one. - [ ] **Runtime-key parity for `run.json` consumers outside `src/`.** The e2e gate in `verify-published-action.yml` reads `task_results[*].status` / `weighted_score` / `total_tokens`, and `action.yml`'s score gate reads `weighted_score` / `task_id`. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c1096c89..45ec37bc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,8 +61,15 @@ jobs: # produced (real release on main, or a stamped prerelease on a branch). version: ${{ steps.ver.outputs.version }} # Real-release-only version: empty on a prerelease dispatch (the `release` - # step is skipped off main). The `promote` job gates on this, so a - # prerelease never moves the major tag or cuts a GitHub Release. + # step is skipped off main), where `version` above instead carries the stamped + # rc -- which is why the two are NOT interchangeable. + # + # What keeps a prerelease from moving the major tag or cutting a Release is the + # `promote` job's `if: github.ref == 'refs/heads/main'`, NOT an emptiness test + # on this output; gating a job on a `needs` output is the skipped-green hazard + # that job's header documents. `promote` consumes this value for the version it + # promotes and enforces non-emptiness INSIDE the job ("Validate version shape"), + # so a lost output is a red job rather than a silent no-op. released_version: ${{ steps.release.outputs.version }} env: # Load-bearing on the release path: the pool enforces a package-age safe-chain @@ -82,6 +89,10 @@ jobs: with: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + # Scoped explicitly: omitting `permission-*` mints a token carrying EVERY + # permission of the installation, and this is the app with the main-branch + # ruleset bypass. All it does here is push the bump commit + tag. + permission-contents: write - name: Checkout code uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -326,12 +337,21 @@ jobs: # Publish the wheel+sdist to public PyPI. This runs as its own job so OIDC # Trusted Publishing is scoped to a dedicated, environment-gated context -- - # no PyPI token/secret is stored. Gated on the release job having actually - # cut a version. + # no PyPI token/secret is stored. + # + # NO `if:` ON THIS JOB, deliberately. It used to carry + # `if: needs.release.outputs.version != ''`, which was both dead and dangerous. Dead: + # "Resolve published version" already `exit 1`s on an empty version, so a successful + # `release` job never produces one. Dangerous: it is the skipped-green shape the + # `promote` header condemns -- if that output failed to carry over into a partial + # "Re-run failed jobs" attempt, this job resolved to SKIPPED, which (since `promote` + # now declares `needs: [release, publish-pypi]`) also skipped the promotion, for a + # fully GREEN run that published no wheel and never moved the major tag. The implicit + # `success()` on `needs: release` is the real gate; emptiness is asserted in-job below, + # so a lost output is RED. publish-pypi: name: Publish to PyPI needs: release - if: needs.release.outputs.version != '' runs-on: uipath-ubuntu-latest timeout-minutes: 10 environment: @@ -341,6 +361,21 @@ jobs: # OIDC token minting for Trusted Publishing; no long-lived credentials. id-token: write steps: + # The enforcement point for a missing version, now that the job's `if:` no longer + # gates on it (see the header). On a successful `release` job this is always set, + # so an empty value means the output did not carry over into a partial re-run -- + # which must be loud, because the alternative shape was a silent skip. + - name: Validate version carried over + env: + VERSION: ${{ needs.release.outputs.version }} + run: | + set -euo pipefail + if [ -z "$VERSION" ]; then + echo "::error title=Release version unavailable::needs.release.outputs.version is empty. On a successful release job it is always set, so the output most likely did not carry over into a partial re-run -- re-run the whole Release workflow's remaining jobs." + exit 1 + fi + echo "publishing coder-eval==$VERSION" + - name: Download built dist uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: @@ -359,8 +394,77 @@ jobs: # job stays permanently red, so `promote` can never run and the major tag is # never moved for a version that IS published -- the stranded state from the # other direction. Trusted-Publishing scoping is unaffected. + # + # It does cost something, which the next step buys back: twine treats PyPI's + # 400 "File already exists" as success WITHOUT comparing content, so on its own + # a green publish stops proving that the wheel THIS run built is the one PyPI + # serves. Before this flag, a duplicate upload failed loudly and incidentally + # established that. Nothing else in release -> promote -> verify re-asserts it + # (promote moves `v0` on job success alone; the nightly preflight checks + # reachability, not identity), so the identity assertion is made explicit below. skip-existing: true + # Re-establish what `skip-existing` gives up: the files PyPI serves for this version + # must be byte-identical to the ones this run built. Without it, a wheel pre-uploaded + # under the release's exact version (compromised maintainer account, leaked legacy + # API token) is silently accepted, `promote` then points `v0` at an action.yml + # pinning it, and every `uses: UiPath/coder_eval@v0` consumer installs it on a fully + # green release. + # + # A mismatch is fatal -- it must stop `promote`. Being unable to READ the index is + # not: the JSON API can lag seconds behind an upload, and a transient must not + # redden a publish that actually succeeded (it would also block the re-run story + # `skip-existing` exists for). So: mismatch => error, unreachable => warning. + - name: Assert PyPI serves this run's artifacts + env: + VERSION: ${{ needs.release.outputs.version }} + run: | + set -euo pipefail + python3 <<'PY' + import hashlib, json, os, pathlib, sys, time, urllib.error, urllib.request + + version = os.environ["VERSION"] + url = f"https://pypi.org/pypi/coder-eval/{version}/json" + + payload = None + for attempt in range(1, 7): + try: + with urllib.request.urlopen(url, timeout=30) as resp: # noqa: S310 - fixed https URL + payload = json.load(resp) + break + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + print(f"attempt {attempt}: could not read {url} ({exc}); waiting for propagation...") + time.sleep(20) + if payload is None: + print(f"::warning title=Artifact identity unverified::could not read {url} after 6 attempts. " + "The publish itself succeeded; this check is inconclusive, not a failure. The nightly " + "Verify Published Action workflow re-checks the pin.") + sys.exit(0) + + remote = {u["filename"]: (u.get("digests") or {}).get("sha256") for u in payload.get("urls") or []} + local = sorted(p for p in pathlib.Path("dist").iterdir() if p.is_file()) + if not local: + print("::error::no files in dist/ to compare -- the download-artifact step produced nothing") + sys.exit(1) + + bad = [] + for path in local: + want = hashlib.sha256(path.read_bytes()).hexdigest() + got = remote.get(path.name) + if got is None: + bad.append(f"{path.name}: not present on PyPI for {version}") + elif got != want: + bad.append(f"{path.name}: PyPI serves sha256 {got}, this run built {want}") + else: + print(f" {path.name}: sha256 matches ({want[:12]}...)") + if bad: + print("::error title=Published artifact is not ours::PyPI does not serve the artifacts this run " + f"built for {version}: " + "; ".join(bad) + ". Do NOT promote: investigate before moving " + "the major tag, since `v0` would point every consumer at these files.") + sys.exit(1) + print(f"PyPI serves exactly the {len(local)} artifact(s) this run built for {version}.") + PY + # Everything CONSUMER-VISIBLE happens here, and only after the wheel is actually # on PyPI: the moving major tag (`v0`, what every consumer pins) and the GitHub # Release (what the Marketplace listing is cut from). @@ -391,6 +495,16 @@ jobs: # re-running publish-pypi. Closing it entirely would mean publishing to PyPI before # pushing any git ref, which requires carrying the bumped commit + tag between jobs # as an artifact; not worth the new failure modes. + # + # NOT COVERED HERE, deliberately: the GHCR agent image. "Build and push versioned + # agent image" stays in the `release` job, pushing `:` and moving `:latest` + # before publish-pypi runs, all under `continue-on-error: true`. So a release whose + # PyPI publish fails still advertises `:latest` for a version absent from PyPI. That + # is accepted rather than overlooked: the image is an INTERNAL convenience (the + # nightly's sandbox base, docs/DOCKER_ISOLATION.md), not a ref a stranger's pipeline + # resolves, and it must be built in the job that holds the bumped pyproject -- moving + # it here would mean re-running buildx and the private-index secrets in a second job + # to protect a best-effort artifact. `v0` is the consumer contract; the image is not. promote: name: Promote major tag and cut GitHub Release needs: [release, publish-pypi] @@ -405,6 +519,11 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 10 + # Every write in this job goes through the app token below, so GITHUB_TOKEN needs + # nothing beyond read. Declared explicitly to drop the workflow-level + # `packages: write`, which exists only for the GHCR steps in the `release` job. + permissions: + contents: read steps: # Pushing the major tag needs the release app's credentials, same as the # version-tag push in the `release` job: the workflow's GITHUB_TOKEN is @@ -415,6 +534,9 @@ jobs: with: app-id: ${{ secrets.RELEASE_APP_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + # Scoped explicitly (see the `release` job's mint): the only writes are the + # major-tag re-point and `gh release create`, both contents. + permission-contents: write # Two jobs in one: (1) the version is interpolated into `ref:` below, so pin its # shape first -- defence-in-depth against a malformed value producing a surprising diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 0b784b78..574c43c1 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -82,7 +82,7 @@ jobs: # here so a 1.0.0 release fails loudly instead of silently leaving the paid # tier testing a stale major forever. if [ "$MAJOR" != "v0" ]; then - echo "::error::major tag is now $MAJOR, but the e2e job below pins @v0. Bump the \`uses:\` in this workflow." + echo "::error::major tag is now $MAJOR, but the e2e job below pins @v0. Bump the \`uses:\` in this workflow AND every doc surface that hardcodes the major: README.md, docs/CI_GATE.md, docs/tutorials/02-ci-pipeline.md (CE026 checks the Marketplace slug, not the major, so it will not catch them)." exit 1 fi @@ -124,6 +124,7 @@ jobs: { echo "pin=$PIN" echo "newest=$VERSION" + echo "major=$MAJOR" echo "lagging=$LAGGING" } >> "$GITHUB_OUTPUT" echo "parity OK: @$MAJOR promises coder-eval==$PIN (newest release $VERSION, lagging=$LAGGING)" @@ -155,10 +156,18 @@ jobs: sleep 20 done - # 000 means we never reached PyPI at all -- that is our problem, not a - # stranded pin, and must not be reported as one. - if [ "$CODE" = "000" ]; then - echo "::error title=PyPI unreachable::could not reach $URL after 6 attempts (curl transport failure). This check is inconclusive -- it does NOT mean coder-eval==${PIN} is missing." + # Only a DEFINITIVE answer from PyPI may be reported as a stranded pin. A + # transport failure or a throttle/outage means we learned nothing, and saying + # "re-run publish-pypi" there sends the operator to re-publish a version that + # is already there (an upload PyPI answers 400 on). Same split the Marketplace + # probe below performs -- these two steps must not disagree about a code. + # 000 -> curl itself failed (DNS/network/TLS). + # 403/429 -> throttled or bot-blocked by the CDN, not an answer about the file. + # 5xx -> PyPI/Fastly transient. + # Still RED either way: this tier's whole job is to prove the pin resolves, and + # an unproven pin must gate the paid tier. Only the diagnosis differs. + if [ "$CODE" = "000" ] || [ "$CODE" = "403" ] || [ "$CODE" = "429" ] || [ "$CODE" -ge 500 ] 2>/dev/null; then + echo "::error title=PyPI check inconclusive::could not get a definitive answer from $URL after 6 attempts (HTTP $CODE -- transport failure, throttling, or a PyPI-side transient). This does NOT mean coder-eval==${PIN} is missing; re-run this workflow. Do NOT re-publish on the strength of this." exit 1 fi echo "::error title=Stranded action.yml pin::coder-eval==${PIN} is NOT on PyPI (HTTP $CODE), but @v0 points at an action.yml that installs it. Every 'uses: UiPath/coder_eval@v0' consumer fails at install. Re-run the Release workflow's publish-pypi job, then its promote job." @@ -180,8 +189,10 @@ jobs: echo "::error title=promote did not run::coder-eval==${NEWEST} is on PyPI but @v0 still promises ${PIN}. The Release workflow's promote job was skipped or failed -- re-run it to move the major tag." exit 1 fi - if [ "$CODE" = "000" ]; then - echo "::warning title=Lag classification inconclusive::could not reach PyPI to check whether ${NEWEST} was published; @v0 still promises ${PIN}, which the previous step verified is installable." + # Same transient split as the step above -- a 403/429/5xx is not evidence that + # ${NEWEST} is unpublished, so it must not be classified as either verdict. + if [ "$CODE" = "000" ] || [ "$CODE" = "403" ] || [ "$CODE" = "429" ] || [ "$CODE" -ge 500 ] 2>/dev/null; then + echo "::warning title=Lag classification inconclusive::PyPI returned $CODE (transport failure, throttling, or transient), so whether ${NEWEST} was published is unknown; @v0 still promises ${PIN}, which the previous step verified is installable." exit 0 fi # Newest is tagged but unpublished => publish-pypi never completed (failed, or @@ -197,16 +208,25 @@ jobs: # consistent with the same `name:`. - name: Verify Marketplace listing resolves env: - # The version tag, which the parity step above proved is the same commit - # the major tag points at. - TAG_REF: v${{ steps.parity.outputs.version }} + # The MAJOR tag, not the newest version tag: this reads what `@v0` + # consumers resolve, and it is the only one guaranteed to exist here. (The + # newest version tag can lag being promoted -- see the parity step -- and in + # that state no Release was cut for it, so the live listing still reflects + # the major tag's commit anyway.) + TAG_REF: ${{ steps.parity.outputs.major }} run: | set -euo pipefail NAME=$(git show "${TAG_REF}:action.yml" \ | sed -nE 's/^name:[[:space:]]*(.+)$/\1/p' | head -1 \ | sed -E 's/^["'"'"']//; s/["'"'"']$//') if [ -z "$NAME" ]; then echo "::error::could not read \`name:\` from action.yml"; exit 1; fi - SLUG=$(echo "$NAME" | tr '[:upper:]' '[:lower:]' | tr ' ' '-') + # Slug derivation, kept on ONE line and anchored: tests/test_verify_published_workflow.py + # extracts this exact line and asserts it agrees with the tested slugger + # (tests/lint/action_docs.py::marketplace_slug, which CE026 uses for the doc links) + # over a table of names. A second, weaker slugger here would 404 on any `name:` + # carrying punctuation or a double space. DO NOT reflow onto multiple lines. + # slug-derivation-anchor + SLUG=$(printf '%s' "$NAME" | tr '[:upper:]' '[:lower:]' | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//; s/[[:space:]]+/-/g; s/[^a-z0-9._-]//g') URL="https://github.com/marketplace/actions/${SLUG}" echo "listing name: $NAME -> $URL" @@ -241,15 +261,18 @@ jobs: uses: astral-sh/setup-uv@38f3f104447c67c051c4a08e39b64a148898af3a # v4.2.0 # Proves the wheel is installable and the console script works -- the exact two - # things the composite action does before it runs anything. - - name: Install from PyPI and smoke the CLI + # things the composite action does before it runs anything. Keyed on the PIN, + # not the newest release: the pin is the consumer contract, and under the + # legitimate lagging state (see the parity step) the newest version may not be + # on PyPI at all while `@v0` consumers are perfectly healthy on the pin. + - name: Install @v0's pinned version from PyPI and smoke the CLI env: - VERSION: ${{ steps.parity.outputs.version }} + PIN: ${{ steps.parity.outputs.pin }} run: | set -euo pipefail - uv tool install "coder-eval==${VERSION}" + uv tool install "coder-eval==${PIN}" coder-eval --help > /dev/null - echo "coder-eval==${VERSION} installs and runs." + echo "coder-eval==${PIN} installs and runs." # TIER 2 -- costs cents. Consumes the action exactly as a stranger would: # `uses: UiPath/coder_eval@v0` with the default `version:` (never `local`), no repo @@ -279,6 +302,15 @@ jobs: uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "20" + # ACCEPTED RISK, deliberate: this install is UNPINNED, and the job forwards + # ANTHROPIC_API_KEY into the run below on a shared GitHub-hosted runner -- so a + # compromised publish of @anthropic-ai/claude-code executes with the key in its + # environment, unattended, nightly. Pinning would defeat a stated purpose of this + # workflow (the header lists that package as drift this nightly exists to catch), + # and the same unpinned install already appears five times in pr-checks.yml, so a + # pin here would buy nothing while the PR path stayed open. Recorded rather than + # silently carried; revisit together with pr-checks.yml if the repo ever adopts a + # pinned agent-runtime install. - name: Install Claude CLI run: npm install -g @anthropic-ai/claude-code @@ -350,25 +382,35 @@ jobs: set -euo pipefail echo "action step outcome: $STEP_OUTCOME (not the gate by itself -- see below)" - # 1. The JUnit report exists at the requested path and is well-formed - # (trusted, self-generated input; our writer emits no DTDs/entities, so - # stdlib ET is fine). + # 1. The JUnit report exists at the requested path (well-formedness and a + # non-vacuous testcase count are asserted in the Python block below, where + # the task_results row count it must match is already parsed). if [ ! -f "$JUNIT" ]; then echo "::error::no JUnit report at $JUNIT. If the action failed during install, the pinned version is probably not installable -- check the preflight job's PyPI result." exit 1 fi - python3 -c "import sys, xml.etree.ElementTree as ET; ET.parse(sys.argv[1])" "$JUNIT" # 1b. Output wiring, as its own assertion with its own message. Only hard-fail - # when the step went green, where propagation is guaranteed; otherwise the - # emptiness is ambiguous (broken action vs. runner not mapping outputs of - # a failed composite) and must not be reported as a broken contract. + # when the step went green, where propagation is guaranteed; otherwise a + # MISSING output is ambiguous (broken action vs. runner not mapping outputs + # of a failed composite) and must not be reported as a broken contract. A + # PRESENT-but-wrong output is not ambiguous at all, though -- no runner + # behavior invents a wrong path -- so it is still checked, as a warning, so + # the wiring contract is never silently unasserted. if [ "$STEP_OUTCOME" = "success" ]; then [ "$OUT_JUNIT" = "$JUNIT" ] || { echo "::error::action's junit-path output is '$OUT_JUNIT', expected '$JUNIT'"; exit 1; } [ "$OUT_RUNDIR" = "$RUNDIR" ] || { echo "::error::action's run-dir output is '$OUT_RUNDIR', expected '$RUNDIR'"; exit 1; } echo "outputs wired correctly." - elif [ -z "$OUT_JUNIT" ] || [ -z "$OUT_RUNDIR" ]; then - echo "::warning::action step was red and its outputs are empty; cannot tell whether the action failed to set them or the runner does not map outputs of a failed composite. Artifact checks below are authoritative." + else + if [ -z "$OUT_JUNIT" ] || [ -z "$OUT_RUNDIR" ]; then + echo "::warning::action step was red and its outputs are empty; cannot tell whether the action failed to set them or the runner does not map outputs of a failed composite. Artifact checks below are authoritative." + fi + if [ -n "$OUT_JUNIT" ] && [ "$OUT_JUNIT" != "$JUNIT" ]; then + echo "::warning::action step was red and its junit-path output is '$OUT_JUNIT', expected '$JUNIT' -- a non-empty wrong value is not explained by output mapping, so the action's output wiring is probably broken." + fi + if [ -n "$OUT_RUNDIR" ] && [ "$OUT_RUNDIR" != "$RUNDIR" ]; then + echo "::warning::action step was red and its run-dir output is '$OUT_RUNDIR', expected '$RUNDIR' -- see above." + fi fi # 3. run.json -- the consumer contract -- exists and describes a real run @@ -376,27 +418,77 @@ jobs: # and the agent runtime actually worked, WITHOUT asserting output quality. [ -f "$RUNDIR/run.json" ] || { echo "::error::run.json missing in $RUNDIR"; exit 1; } RUN_JSON="$RUNDIR/run.json" python3 <<'PY' - import json, os, sys + import json, os, sys, xml.etree.ElementTree as ET + + # Error categories that mean the failure is UPSTREAM of the published action -- + # the model/API was unavailable or the agent process died on a transient. Those + # must not be reported as broken wiring: this check answers "does the published + # action still work", not "is the model available right now". Everything else + # (auth, billing, config, sandbox/install, or no category at all) IS the + # published action's problem and stays a hard error. + UPSTREAM = {"agent_api_error", "agent_rate_limit", "agent_timeout", "agent_crash"} data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) rows = data.get("task_results") or [] if not rows: print("::error::run.json contains no task_results -- the action produced no run") sys.exit(1) + + # The JUnit report must describe the same run, not merely be parseable: an empty + # but well-formed passed the old parse-only check. Trusted, + # self-generated input (our writer emits no DTDs/entities), so stdlib ET is fine. + # `>=` not `==`: reports_junit.py also emits synthetic `skipped` / `suite-gates` + # testsuites, which only ever ADD cases. + cases = len(list(ET.parse(os.environ["JUNIT"]).getroot().iter("testcase"))) + if cases < len(rows): + print(f"::error::JUnit report has {cases} element(s) for {len(rows)} task_results row(s) " + "-- the action's JUnit conversion is broken for consumers") + sys.exit(1) # Key is "status" (eval_result_to_task_dict writes FinalStatus there); there is # no "final_status" key in run.json rows, and reading one would silently # evaluate to None and make the exit-contract assertion below dead code. statuses = [r.get("status") for r in rows] + categories = [r.get("error_category") for r in rows] tokens = sum(r.get("total_tokens") or 0 for r in rows) for r in rows: print(f" {r.get('task_id')}: status={r.get('status')} " - f"score={r.get('weighted_score')} tokens={r.get('total_tokens')}") - + f"score={r.get('weighted_score')} tokens={r.get('total_tokens')} " + f"error_category={r.get('error_category')}") + + # Zero tokens means no generation was ever billed. That IS the wiring signal + # this gate exists for -- but `total_tokens` is also empty when every turn died + # before a usage record existed (a sustained 429/529, a sandbox-setup failure), + # which is upstream and not actionable by us. On a daily cron the transient case + # will eventually occur, and sending the operator to audit credential + # passthrough for an Anthropic outage is how a check earns being ignored. So + # discriminate on the category run.json already records per row. if tokens <= 0: - print("::error::no tokens consumed across any task -- the agent never reached the model " + if any(c in UPSTREAM for c in categories): + print("::warning::no tokens consumed, but the run reports an upstream failure " + f"(error_category={[c for c in categories if c in UPSTREAM]}) -- the model/API was " + "unavailable, which says nothing about the published action. Inconclusive, not a " + "wiring failure; re-run to confirm.") + sys.exit(0) + print("::error::no tokens consumed across any task and no upstream error category " + f"(statuses={statuses}, categories={categories}) -- the agent never reached the model " "(credential passthrough, agent runtime, or backend wiring is broken)") sys.exit(1) + # A harness/environment error is NOT a model flake, and must not be tolerated as + # one just because the step is continue-on-error. FinalStatus.ERROR and + # BUILD_FAILED are exactly the statuses models/enums.py maps to the "error" + # reporting category -- i.e. something broke around the agent rather than the + # agent doing poorly -- so they are a "published action is broken" signal unless + # the category says the cause was upstream. + harness_errors = [ + (s, c) for s, c in zip(statuses, categories, strict=True) + if s in {"ERROR", "BUILD_FAILED"} and c not in UPSTREAM + ] + if harness_errors: + print(f"::error::task(s) failed with a harness/environment error, not a model flake: " + f"{harness_errors} -- the published action's install, sandbox, or agent wiring is broken") + sys.exit(1) + # Exit-contract check, conditional on the model having actually performed. # Ignoring the step's exit code entirely (see the continue-on-error rationale # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a @@ -414,8 +506,12 @@ jobs: f"statuses={statuses}") PY - - name: Upload run on failure - if: failure() + # `always()`, not `failure()`: the action step is continue-on-error, so the routine + # tolerated-red case the gate is designed around (a model flake failing + # `file_exists`) leaves the JOB green -- and `failure()` would then discard the run + # dir that is the only evidence explaining the flake. + - name: Upload run + if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: verify-published-runs diff --git a/CLAUDE.md b/CLAUDE.md index 7ca54ef8..4794b5f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,7 +124,7 @@ docs/ # Documentation templates/ # Sandbox template directories .claude-plugin/marketplace.json # Makes this repo a Claude Code plugin marketplace (`/plugin marketplace add UiPath/coder_eval`); lists the one plugin below. plugins/coder-eval/ # The published Claude Code plugin: `.claude-plugin/plugin.json` (its `version` is a derived pin of pyproject's, bumped by release.yml, guarded by tests/test_action_version_pin.py), `skills//SKILL.md` × 6 (`/coder-eval:init`, `/coder-eval:check-skill`, `/coder-eval:task`, `/coder-eval:lint-tasks`, `/coder-eval:analyze`, `/coder-eval:ci`), and `reference/` — everything a skill reads must live here, since an installed plugin is copied to ~/.claude/plugins/cache/ WITHOUT its parent dirs (address it via `${CLAUDE_PLUGIN_ROOT}`). `reference/criteria.md` is generated (`make plugin-reference`, CE033); `reference/run-layout.md` is a verbatim mirror of `.claude/shared/run-layout.md`; `reference/task-rubric.md` is the shared task-quality rubric that `task` and `lint-tasks` both read (plugin-only — no repo-side twin); `reference/repo-layout.md` is the eval-tree DISCOVERY policy every skill reads (`SKILL_NEEDS_EVAL_ROOT_DISCOVERY`, which a new skill must declare a stance in) — glob for `task_id:` files and `run.json`, never assume `tasks/`/`runs/latest` — as distinct from `run-layout.md`, which describes what is inside a run directory. Every skill must appear in all four surfaces in `SKILL_DOC_SURFACES` (derived test), and their combined frontmatter `description` length is capped (`SKILL_LISTING_BUDGET_CHARS`) because the skill listing's budget is shared with every skill the user has installed. **Skill naming is verb-first imperative** — a skill is a command you issue (`/coder-eval:`) and every one of them takes an action, so name it for the action: a bare verb where that is unambiguous (`init`, `analyze` — the object comes from the argument), otherwise `-` (`lint-tasks`, `check-skill`). Never `-`: `skill-check` was renamed to `check-skill` precisely because it read backwards next to `lint-tasks`. `task` and `ci` predate the rule and stay — renaming a published skill breaks every user's muscle memory for no functional gain, since activation keys on the `description`, never the name. Distinct from `.claude/commands/`, which stays repo-local contributor tooling. -action.yml # Published composite GitHub Action (coder-eval as a CI gate). release.yml maintains its `version:` default + the moving `v` tag. +action.yml # Published composite GitHub Action (coder-eval as a CI gate). release.yml's `release` job maintains its `version:` default; its `promote` job (gated on publish-pypi) moves the `v` tag + cuts the Release, so nothing consumer-visible moves before the wheel is on PyPI. verify-published-action.yml then verifies the published composite (tag/pin/PyPI/Marketplace parity, plus a real consumer run) after each Release and nightly. Runbook: CONTRIBUTING.md § Releasing. ``` ## Key Architectural Patterns diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 62d80cb9..82cfe808 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -122,6 +122,58 @@ behind the published Action and must exercise the image integrators actually use - Tests should use **Haiku or at most Sonnet** for any model calls — never Opus (cost). +## Releasing + +Merges to `main` do **not** release. Dispatch the **Release** workflow from the +Actions tab and pick a bump level. It runs three jobs, in this order: + +| Job | Does | Re-runnable? | +|-----|------|--------------| +| `release` | bumps the version, bumps `action.yml`'s `version:` pin, tags `vX.Y.Z`, pushes `main` + that tag, builds the wheel/sdist, pushes the GHCR agent image | **No** — re-running bumps and tags a *second* version | +| `publish-pypi` | publishes the wheel/sdist to public PyPI (OIDC, `pypi` environment) and asserts PyPI serves the exact files this run built | Yes | +| `promote` | moves the `v0` tag and cuts the GitHub Release | Yes | + +`v0` is the ref every consumer pins (`uses: UiPath/coder_eval@v0`), and the +composite action installs `coder-eval==`. So **nothing +consumer-visible moves until the wheel is on PyPI** — that is why the tag move and +the Release live in `promote` rather than in `release`. + +### When a release goes red + +Recover by re-running the failed jobs, never the whole workflow (`release` is not +re-runnable). `v0` keeps pointing at the last fully-published release throughout. + +- **`publish-pypi` failed or is waiting on the `pypi` environment approval** — + `vX.Y.Z` and `main` reference a version not yet on PyPI, but `@v0` consumers are + healthy on the previous release. Re-run `publish-pypi`, then `promote`. +- **`promote` failed** — the wheel is published but `v0` still promises the previous + version. Re-run `promote`. +- **`promote` refuses with "Refusing to move v0 backwards"** — you are re-running an + *older* release's promote (GitHub keeps re-run available for 30 days). Promote the + newest tag instead; the guard exists because a force-move would downgrade every + consumer. +- **"Published artifact is not ours"** — PyPI serves files this run did not build. + Do not promote; investigate before `v0` points consumers at them. + +### The nightly gate + +**Verify Published Action** (`.github/workflows/verify-published-action.yml`) runs +after every Release, on a daily cron, and on demand. Tier 1 is free and +deterministic; tier 2 spends a few cents driving the published action as a stranger +would. Annotations it emits, and what each means: + +| Annotation | Meaning | +|---|---| +| `Stranded action.yml pin` | `@v0` promises a version PyPI does not have. Consumers are broken **now**. Re-run `publish-pypi`, then `promote`. | +| `promote did not run` | the newest version is published but `v0` still promises the previous one. Re-run `promote`. | +| `Release incomplete` (warning) | newest version tagged but unpublished; `@v0` consumers are fine. Finish the release. | +| `PyPI check inconclusive` / `Marketplace check inconclusive` / `Lag classification inconclusive` | an upstream transient or throttle, not a verdict. Re-run; do **not** re-publish on the strength of it. | +| `Marketplace listing missing` | the listing was renamed or delisted, or `action.yml`'s `name:` changed without it following. | +| `no tokens consumed … wiring is broken` | the published action never reached the model — credentials, agent runtime, or backend. | +| `harness/environment error` | a task failed for a non-model reason (install, sandbox, config). | + +There is no notification path: a red nightly appears only in the Actions tab. + ## License By contributing, you agree that your contributions will be licensed under the diff --git a/tests/lint/workflow_outputs.py b/tests/lint/workflow_outputs.py new file mode 100644 index 00000000..fb7b0672 --- /dev/null +++ b/tests/lint/workflow_outputs.py @@ -0,0 +1,250 @@ +"""CE035 — every ``steps..outputs.`` / ``needs..outputs.`` reference +in a workflow must resolve to a key its writer actually produces. + +The motivating bug shipped in ``verify-published-action.yml``: two steps read +``steps.parity.outputs.version``, but the ``parity`` step writes only ``pin`` / +``newest`` / ``lagging`` (the *shell variable* was ``VERSION``, the *output key* was +``newest``). GitHub expands an unwritten output to the empty string, so +``TAG_REF: v${{ steps.parity.outputs.version }}`` became the bare string ``v``, +``git show "v:action.yml"`` exited 128 under ``set -euo pipefail``, and the preflight +job was red on 100% of triggers — which, via ``needs: preflight``, meant the paid +end-to-end tier could never run at all. + +Nothing caught it: the workflow is invisible to ruff, pyright, pytest and the AST lint +runner, and ``actionlint`` models ``steps.*.outputs`` as an open string map, so an +unwritten shell key is untyped and unflagged there too. + +**Writers are mechanically enumerable, and this rule only reasons about the ones that +are.** For a referenced step id: + +* ``run:`` step → the keys it echoes/prints into ``$GITHUB_OUTPUT``. Writers are + collected by an over-approximating scan (any ``key=`` / ``key<<`` in an ``echo`` or + ``printf`` in the body), because over-approximating *writers* can only make the rule + quieter, never produce a false failure. If a body touches ``$GITHUB_OUTPUT`` in a way + the scan cannot read (no key found at all), the step is skipped rather than guessed at. +* local composite (``uses: ./``) → the ``outputs:`` block of the repo's ``action.yml``. +* third-party ``uses:`` → **skipped**. Resolving those needs the action's own metadata, + which is not on disk; pretending otherwise would fail on every pinned action. +* a missing step id, or a ``needs`` output absent from that job's ``outputs:`` map, is + always a finding — those are fully enumerable from the file. + +Like CE026-CE031 this is deliberately NOT a ``BaseRule`` in ``tests/lint/runner.py``: +that runner is AST-only over ``.py`` files, whereas this rule reasons over workflow YAML +plus embedded shell. It is wired as ``tests/test_custom_lint.py::TestCE035WorkflowOutputParity``. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + + +# `${{ steps..outputs. }}` — the id and key charsets GitHub accepts. +STEP_OUTPUT_REF = re.compile(r"steps\.(?P[A-Za-z_][A-Za-z0-9_-]*)\.outputs\.(?P[A-Za-z_][A-Za-z0-9_.-]*)") +NEEDS_OUTPUT_REF = re.compile(r"needs\.(?P[A-Za-z_][A-Za-z0-9_-]*)\.outputs\.(?P[A-Za-z_][A-Za-z0-9_.-]*)") + +# `echo "key=value"` / `printf 'key=%s' …` / `echo "key<[A-Za-z_][A-Za-z0-9_.-]*)(?:=|<<)""") + + +@dataclass(frozen=True) +class Finding: + """One unresolvable output reference.""" + + path: Path + line: int + message: str + + def __str__(self) -> str: + return f"{self.path}:{self.line} — {self.message}" + + +def workflow_paths(repo_root: Path) -> list[Path]: + """Every workflow file, plus the composite action definition.""" + paths = sorted(p for p in (repo_root / ".github" / "workflows").glob("*.yml") if p.is_file()) + action = repo_root / "action.yml" + if action.is_file(): + paths.append(action) + return paths + + +def _iter_strings(node: Any) -> list[str]: + """Every string anywhere in a parsed YAML subtree.""" + if isinstance(node, str): + return [node] + if isinstance(node, dict): + return [s for v in node.values() for s in _iter_strings(v)] + if isinstance(node, list): + return [s for v in node for s in _iter_strings(v)] + return [] + + +def _first_line_containing(lines: list[str], needle: str) -> int: + for i, line in enumerate(lines, start=1): + if needle in line: + return i + return 1 + + +def _local_composite_outputs(repo_root: Path) -> set[str]: + action = repo_root / "action.yml" + if not action.is_file(): + return set() + data = yaml.safe_load(action.read_text(encoding="utf-8")) or {} + return set((data.get("outputs") or {}).keys()) + + +def _written_keys(step: dict[str, Any]) -> set[str] | None: + """Output keys a ``run:`` step writes, or ``None`` when they are not determinable.""" + body = step.get("run") + if not isinstance(body, str): + return None + if "GITHUB_OUTPUT" not in body: + return set() + keys = {m.group("key") for m in OUTPUT_WRITE.finditer(body)} + # A body that clearly writes outputs but yields no readable key (e.g. built by an + # embedded interpreter) is unparseable, not empty — skip rather than guess. + return keys or None + + +def _steps_of(job: dict[str, Any]) -> list[dict[str, Any]]: + steps = job.get("steps") + return [s for s in steps if isinstance(s, dict)] if isinstance(steps, list) else [] + + +def _check_step_refs( + path: Path, + lines: list[str], + scope_name: str, + steps: list[dict[str, Any]], + scope_strings: list[str], + composite_outputs: set[str], +) -> list[Finding]: + findings: list[Finding] = [] + by_id = {s["id"]: s for s in steps if isinstance(s.get("id"), str)} + seen: set[tuple[str, str]] = set() + + for text in scope_strings: + for match in STEP_OUTPUT_REF.finditer(text): + step_id, key = match.group("id"), match.group("key") + if (step_id, key) in seen: + continue + seen.add((step_id, key)) + line = _first_line_containing(lines, match.group(0)) + + step = by_id.get(step_id) + if step is None: + findings.append( + Finding( + path, + line, + f"{scope_name}: `steps.{step_id}.outputs.{key}` refers to step id " + f"'{step_id}', which does not exist in this job " + f"(ids present: {sorted(by_id) or 'none'})", + ) + ) + continue + + uses = step.get("uses") + if isinstance(uses, str): + if not uses.startswith("./"): + continue # third-party action: outputs are not on disk — see docstring + if key not in composite_outputs: + findings.append( + Finding( + path, + line, + f"{scope_name}: `steps.{step_id}.outputs.{key}` — the local composite " + f"action declares outputs {sorted(composite_outputs)}", + ) + ) + continue + + written = _written_keys(step) + if written is None: + continue # not a shell step, or writers not statically readable + if key not in written: + findings.append( + Finding( + path, + line, + f"{scope_name}: `steps.{step_id}.outputs.{key}` is never written — step " + f"'{step_id}' writes {sorted(written) or 'no outputs'} to $GITHUB_OUTPUT. " + "GitHub expands an unwritten output to the empty string, so this silently " + "becomes ''", + ) + ) + return findings + + +def _check_needs_refs( + path: Path, + lines: list[str], + jobs: dict[str, Any], +) -> list[Finding]: + findings: list[Finding] = [] + seen: set[tuple[str, str]] = set() + declared = {name: set((job.get("outputs") or {}).keys()) for name, job in jobs.items() if isinstance(job, dict)} + for job_name, job in jobs.items(): + if not isinstance(job, dict): + continue + for text in _iter_strings(job): + for match in NEEDS_OUTPUT_REF.finditer(text): + producer, key = match.group("job"), match.group("key") + if (producer, key) in seen: + continue + seen.add((producer, key)) + if producer not in declared: + continue # unknown job name — actionlint's territory, not this rule's + if key not in declared[producer]: + findings.append( + Finding( + path, + _first_line_containing(lines, match.group(0)), + f"job '{job_name}': `needs.{producer}.outputs.{key}` is not declared — job " + f"'{producer}' exposes {sorted(declared[producer]) or 'no outputs'}. It " + "expands to the empty string, so an emptiness gate on it silently skips", + ) + ) + return findings + + +def find_unresolved_output_refs(paths: list[Path], repo_root: Path) -> list[Finding]: + """Every output reference in ``paths`` that cannot resolve to a real writer.""" + composite_outputs = _local_composite_outputs(repo_root) + findings: list[Finding] = [] + + for path in paths: + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + data = yaml.safe_load(text) or {} + if not isinstance(data, dict): + continue + + jobs = data.get("jobs") + if isinstance(jobs, dict): + for name, job in jobs.items(): + if not isinstance(job, dict): + continue + findings.extend( + _check_step_refs( + path, lines, f"job '{name}'", _steps_of(job), _iter_strings(job), composite_outputs + ) + ) + findings.extend(_check_needs_refs(path, lines, jobs)) + + # A composite action definition (`action.yml`) has one flat step list, and its + # own `outputs:` block reads from those steps. + runs = data.get("runs") + if isinstance(runs, dict) and isinstance(runs.get("steps"), list): + scope_strings = _iter_strings(runs) + _iter_strings(data.get("outputs") or {}) + findings.extend( + _check_step_refs(path, lines, "composite", _steps_of(runs), scope_strings, composite_outputs) + ) + + return findings diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 9f923a02..e3f79983 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2760,3 +2760,96 @@ def test_diff_all_reports_a_missing_file_as_full_drift(self, tmp_path: Path): findings = diff_all({target: "body\n"}) assert list(findings) == [str(target)] assert "+body" in findings[str(target)] + + +@pytest.mark.lint +class TestCE035WorkflowOutputParity: + """CE035 — a `steps..outputs.` / `needs..outputs.` reference must + resolve to a key its writer actually produces. + + The motivating bug: `verify-published-action.yml` read + `steps.parity.outputs.version` twice, but that step writes `pin`/`newest`/`lagging` + (the shell *variable* was `VERSION`, the output *key* was `newest`). GitHub expands an + unwritten output to '', so `TAG_REF: v${{ … }}` became the bare `v`, `git show + "v:action.yml"` exited 128 under `set -euo pipefail`, and the preflight job was red on + 100% of triggers — taking the paid e2e tier (`needs: preflight`) with it. Invisible to + ruff/pyright/pytest, and actionlint models `steps.*.outputs` as an open string map. + Reasons over workflow YAML + embedded shell, so it lives here, not in the AST runner. + """ + + REPO_ROOT = Path(__file__).parent.parent + + def test_all_workflow_output_refs_resolve(self): + from tests.lint.workflow_outputs import find_unresolved_output_refs, workflow_paths + + findings = find_unresolved_output_refs(workflow_paths(self.REPO_ROOT), self.REPO_ROOT) + assert not findings, "unresolvable workflow output references:\n" + "\n".join(f" {f}" for f in findings) + + def test_catches_an_unwritten_step_output(self, tmp_path: Path): + """The exact shape of the shipped bug: reading a key the writer never echoes.""" + from tests.lint.workflow_outputs import find_unresolved_output_refs + + wf = tmp_path / ".github" / "workflows" + wf.mkdir(parents=True) + (wf / "w.yml").write_text( + "jobs:\n" + " j:\n" + " steps:\n" + " - id: parity\n" + " run: |\n" + " {\n" + ' echo "pin=$PIN"\n' + ' echo "newest=$VERSION"\n' + ' } >> "$GITHUB_OUTPUT"\n' + " - env:\n" + " TAG_REF: v${{ steps.parity.outputs.version }}\n" + " PIN_REF: ${{ steps.parity.outputs.pin }}\n" + " run: echo hi\n", + encoding="utf-8", + ) + findings = find_unresolved_output_refs([wf / "w.yml"], tmp_path) + assert len(findings) == 1, [str(f) for f in findings] + assert "outputs.version` is never written" in findings[0].message + assert "['newest', 'pin']" in findings[0].message + + def test_catches_a_missing_step_id_and_an_undeclared_needs_output(self, tmp_path: Path): + from tests.lint.workflow_outputs import find_unresolved_output_refs + + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" + " a:\n" + " outputs:\n" + " version: ${{ steps.ver.outputs.version }}\n" + " steps:\n" + " - id: ver\n" + ' run: echo "version=1.2.3" >> "$GITHUB_OUTPUT"\n' + " b:\n" + " if: needs.a.outputs.released_version != ''\n" + " steps:\n" + " - run: echo ${{ steps.nope.outputs.x }}\n", + encoding="utf-8", + ) + messages = [f.message for f in find_unresolved_output_refs([wf], tmp_path)] + assert any("does not exist in this job" in m for m in messages), messages + assert any("needs.a.outputs.released_version` is not declared" in m for m in messages), messages + + def test_skips_third_party_actions_and_unreadable_writers(self, tmp_path: Path): + """Boundaries that keep the rule sound: a pinned action's outputs are not on disk, + and a body that writes outputs from an embedded interpreter is not guessed at.""" + from tests.lint.workflow_outputs import find_unresolved_output_refs + + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" + " j:\n" + " steps:\n" + " - id: app-token\n" + " uses: actions/create-github-app-token@abc123\n" + " - id: py\n" + " run: |\n" + ' python3 -c \'import os; open(os.environ["GITHUB_OUTPUT"], "a")\'\n' + " - run: echo ${{ steps.app-token.outputs.token }} ${{ steps.py.outputs.whatever }}\n", + encoding="utf-8", + ) + assert find_unresolved_output_refs([wf], tmp_path) == [] diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py new file mode 100644 index 00000000..16e3d348 --- /dev/null +++ b/tests/test_verify_published_workflow.py @@ -0,0 +1,217 @@ +"""``verify-published-action.yml`` couples to things nothing else asserts. + +The workflow cannot be exercised before merge — ``workflow_run`` and ``schedule`` only +fire from the default branch — so every coupling it makes to another file is a place +where a rename passes ``make verify`` green and the gate silently rots in production. +Four such couplings, each with an executable binding here: + +1. **``workflow_run: workflows: ["Release"]``** matches ``release.yml``'s ``name:`` by + display string. GitHub does not error on an unmatched name; the trigger simply never + fires, degrading the gate to schedule-only with no signal. +2. **The Marketplace slug** is derived by a shell pipeline, a *second* slugger next to + the tested ``tests/lint/action_docs.py::marketplace_slug`` that CE026 uses for the doc + links. They agree today only because ``action.yml``'s ``name:`` is ``coder_eval`` — the + one input for which both are the identity function. +3. **The ``# <-- kept in sync`` pin anchor** now has three readers with three different + whitespace tolerances (``release.yml``'s sed, this workflow's sed, and + ``tests/test_action_version_pin.py``). A reformat can leave one reporting "parity OK" + on a pin another silently refused to bump. +4. **The inline consumer task YAML** is a whole ``TaskDefinition`` document that no test + validates, while CE029 already validates that exact shape in Markdown. Any field + rename (or an ``extra="forbid"`` violation) would surface only as an opaque failure in + the paid nightly. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from tests.lint.action_docs import action_listing_name, marketplace_slug + + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = REPO_ROOT / ".github" / "workflows" +VERIFY_WF = WORKFLOWS / "verify-published-action.yml" +RELEASE_WF = WORKFLOWS / "release.yml" +ACTION_YML = REPO_ROOT / "action.yml" + + +def _load(path: Path) -> dict[str, Any]: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + assert isinstance(data, dict), f"{path} did not parse as a mapping" + return data + + +def _triggers(workflow: dict[str, Any]) -> dict[str, Any]: + """The ``on:`` block. PyYAML resolves the bare key ``on`` to the boolean ``True``.""" + block = workflow.get("on", workflow.get(True)) + assert isinstance(block, dict), "workflow has no parseable `on:` block" + return block + + +def _run_body(workflow: dict[str, Any], step_name: str) -> str: + """The ``run:`` script of a named step, already dedented by the YAML parser.""" + for job in workflow["jobs"].values(): + for step in job.get("steps") or []: + if isinstance(step, dict) and step.get("name") == step_name: + body = step.get("run") + assert isinstance(body, str), f"step '{step_name}' has no `run:` body" + return body + raise AssertionError(f"no step named '{step_name}'") + + +def _line_after(body: str, anchor: str) -> str: + lines = body.splitlines() + for i, line in enumerate(lines): + if anchor in line: + assert i + 1 < len(lines), f"anchor '{anchor}' is the last line of the step" + return lines[i + 1].strip() + raise AssertionError(f"anchor '{anchor}' not found — did the step get reflowed?") + + +def _bash(script: str, stdin: str = "", env: dict[str, str] | None = None) -> str: + """Run a snippet lifted verbatim out of a workflow. Inputs go through the + environment, never argv or interpolation, so a fixture value carrying quotes cannot + be mistaken for shell syntax.""" + proc = subprocess.run( + ["bash", "-c", script], + input=stdin, + capture_output=True, + text=True, + encoding="utf-8", + env={**os.environ, **(env or {})}, + check=False, + ) + assert proc.returncode == 0, f"script failed ({proc.returncode}): {proc.stderr}" + return proc.stdout + + +def _slug_pipeline() -> str: + """The one-line slug derivation in the preflight job, lifted from its anchor.""" + body = _run_body(_load(VERIFY_WF), "Verify Marketplace listing resolves") + pipeline = _line_after(body, "slug-derivation-anchor") + assert pipeline.startswith("SLUG="), f"unexpected line under the anchor: {pipeline!r}" + return pipeline + + +# -------------------------------------------------------------------------------------- +# 1. workflow_run couples to release.yml's display name +# -------------------------------------------------------------------------------------- + + +def test_workflow_run_names_the_real_release_workflow(): + named = _triggers(_load(VERIFY_WF))["workflow_run"]["workflows"] + release_name = _load(RELEASE_WF)["name"] + assert named == [release_name], ( + f"verify-published-action.yml triggers on workflows {named}, but release.yml is named " + f"'{release_name}'. GitHub does not error on an unmatched name — the trigger just never " + "fires, so release-time verification degrades to the nightly cron with no signal." + ) + + +# -------------------------------------------------------------------------------------- +# 2. one Marketplace slugger +# -------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "listing_name", + [ + "coder_eval", # today's value: the one input where every slugger agrees + "Coder Eval (CI gate)", # punctuation the naive `tr ' ' '-'` pipeline kept + " Coder Eval ", # leading/trailing space + a whitespace run + "coder_eval v2.0", # a dot, which is legal in a slug + ], +) +def test_workflow_slug_pipeline_matches_the_tested_slugger(listing_name: str): + got = _bash( + f'NAME="$LISTING_NAME"; {_slug_pipeline()}; printf "%s" "$SLUG"', + env={"LISTING_NAME": listing_name}, + ) + assert got == marketplace_slug(listing_name), ( + f"the workflow's slug pipeline yields {got!r} for {listing_name!r} but " + f"marketplace_slug() (which CE026 pins the doc links to) yields " + f"{marketplace_slug(listing_name)!r}" + ) + + +def test_action_listing_name_slugs_identically_in_both_implementations(): + """The live value, end to end: whatever `action.yml` declares today.""" + name = action_listing_name(ACTION_YML) + got = _bash( + f'NAME="$LISTING_NAME"; {_slug_pipeline()}; printf "%s" "$SLUG"', + env={"LISTING_NAME": name}, + ) + assert got == marketplace_slug(name) + + +# -------------------------------------------------------------------------------------- +# 3. the `# <-- kept in sync` pin anchor has three readers +# -------------------------------------------------------------------------------------- + + +def test_all_three_pin_anchor_readers_agree_on_action_yml(): + """release.yml's sed (bump), this workflow's sed (read), and the unit test's regex.""" + from tests.test_action_version_pin import _PIN_PATTERN + + action_text = ACTION_YML.read_text(encoding="utf-8") + expected = _PIN_PATTERN.search(action_text) + assert expected is not None, "the pin anchor regex no longer matches action.yml" + + # (a) The workflow's EXTRACTING sed, applied to action.yml exactly as the preflight + # job applies it to `git show v0:action.yml`. + read_sed = next( + line.strip() + for line in _run_body(_load(VERIFY_WF), "Check tag / pin parity").splitlines() + if line.strip().startswith("| sed -nE") and "kept in sync" in line + ).lstrip("| ") + # The sed closes the `PIN=$(git show … | sed …)` substitution the parity step opens. + read_sed = read_sed.removesuffix(")") + read = _bash(read_sed, stdin=action_text).strip() + assert read == expected.group("version"), ( + f"the workflow's sed reads the pin as {read!r} but the anchor regex reads {expected.group('version')!r}" + ) + + # (b) release.yml's BUMPING sed. `-i` and the filename are dropped so the expression + # is exercised portably over stdin (BSD sed's `-i` takes a suffix argument). + bump_sed = next( + line.strip() + for line in _run_body( + _load(RELEASE_WF), "Regenerate uv.lock, bump action.yml pin, and amend release commit" + ).splitlines() + if line.strip().startswith("sed -i -E") and "kept in sync" in line + ) + bump_sed = bump_sed.replace("sed -i -E", "sed -E").removesuffix(" action.yml") + bumped = _bash(f"VERSION=9.9.9; {bump_sed}", stdin=action_text) + assert 'default: "9.9.9"' in bumped, ( + "release.yml's sed did not match the pin anchor in action.yml, so a release would " + "ship a stale `version:` default (its own grep guard would fail the release)" + ) + + +# -------------------------------------------------------------------------------------- +# 4. the inline consumer task YAML is a real TaskDefinition +# -------------------------------------------------------------------------------------- + + +def test_inline_consumer_task_yaml_loads(tmp_path: Path): + from coder_eval.orchestration.task_loader import load_task + + body = _run_body(_load(VERIFY_WF), "Write a consumer task YAML") + lines = body.splitlines() + start = next(i for i, line in enumerate(lines) if "<<'YAML'" in line) + end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "YAML") + task_yaml = "\n".join(lines[start + 1 : end]) + "\n" + + path = tmp_path / "published_smoke.yaml" + path.write_text(task_yaml, encoding="utf-8") + task, _ = load_task(path) + + assert task.task_id == "published_action_smoke" + assert task.success_criteria, "the nightly's task must assert something" From 1cd40832380171e7b1c054648fcf39292a2c531d Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 13 Aug 2026 08:50:07 -0700 Subject: [PATCH 5/9] chore: reconcile the published-action verification with main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Semantic fallout of rebasing onto main (plugin ship + managed runner pool), none of it caught by the textual merge: * release.yml renamed the pin-bump step to "... action.yml + plugin.json pins", which test_verify_published_workflow.py binds by name — the anchor-parity test was failing on a stale literal. * main moved CI to the uipath-* managed pool. `promote` and `preflight` follow; `e2e` deliberately stays on stock ubuntu-latest, with the reason recorded at its own runs-on, because it exists to reproduce what the documented consumer snippet gets. * CExxx ids collided: main shipped CE034 (armed-positive) and this branch ships CE035 (workflow output parity), while both sides had minted candidates under those numbers. Renumbered the three candidates to CE038/CE039/CE041 and listed CE035 among the whole-tree rules in CLAUDE.md. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 10 +++++----- .github/workflows/release.yml | 2 +- .github/workflows/verify-published-action.yml | 6 +++++- CLAUDE.md | 2 +- tests/test_verify_published_workflow.py | 2 +- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index c19a121a..b93c818d 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -187,7 +187,7 @@ harness once for all of them. mechanizable guard; the durable lesson is: when a test names a narrowing, construct the fixture so the row SURVIVES every other rule, or the assertion proves nothing. -- [ ] **CE034 — runner-label registry + dogfood runner parity** over +- [ ] **CE038 — runner-label registry + dogfood runner parity** over `.github/workflows/*.yml`. Two clauses: (a) every label a job can land on must appear in `.github/actionlint.yaml`'s `self-hosted-runner.labels` or a stock GitHub-hosted allowlist — including *both* branches of an expression-valued `runs-on:`, which @@ -247,7 +247,7 @@ harness once for all of them. ## From the PR #82 review follow-up (2026-08-10) -- [ ] **CE035 — documented `coder-eval` invocations must be executable as written.** +- [ ] **CE039 — documented `coder-eval` invocations must be executable as written.** `init/SKILL.md` told the agent to run `coder-eval plan ` and "iterate until it exits 0", which the CLI rejects outright (`plan` takes files; a directory argument exits 1 with a hint) — an unreachable loop condition @@ -260,12 +260,12 @@ harness once for all of them. that duplicates the CLI signature. Deferred on that split — caught in the PR #82 review, fixed by hand in `init/SKILL.md`. -- [ ] **Documented-CLI live smoke.** The behavioural counterpart to CE035: in a +- [ ] **Documented-CLI live smoke.** The behavioural counterpart to CE039: in a `-m live`/`-m slow` test, materialize a fixture repo with one task YAML and execute every fenced `coder-eval …` command extracted from the shipped skills and docs, asserting exit 0 (or an explicitly-expected non-zero). This is the only form that proves argument shape rather than command existence. Not - statically reachable, hence separate from CE035 — proposed in the PR #82 review. + statically reachable, hence separate from CE039 — proposed in the PR #82 review. ## From the 2026-08-11 plugin generic-adopter run @@ -394,7 +394,7 @@ with the two `action.yml` items above — one considered change to the action's ## From 2026-08-04 published-action verification review -- [ ] **CE034 — `VAR=$(… | grep …)` under `set -e` followed by an emptiness check +- [ ] **CE041 — `VAR=$(… | grep …)` under `set -e` followed by an emptiness check is a dead diagnostic.** With `set -euo pipefail`, a pipeline whose `grep` matches nothing exits 1, so the assignment aborts the step *before* the `if [ -z "$VAR" ]; then echo "::error::…"` branch that was written to report it — diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45ec37bc..c8f9aaeb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -517,7 +517,7 @@ jobs: # and no Release is cut. Emptiness is enforced inside the job instead, by # "Validate version shape", so a lost output is a RED job, not a silent no-op. if: github.ref == 'refs/heads/main' - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 10 # Every write in this job goes through the app token below, so GITHUB_TOKEN needs # nothing beyond read. Declared explicitly to drop the workflow-level diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 574c43c1..8949cabc 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -42,7 +42,7 @@ jobs: # tag/pin/PyPI desync, so it gates the paid tier below. preflight: name: Preflight (tag/pin/PyPI/Marketplace parity) - runs-on: ubuntu-latest + runs-on: uipath-ubuntu-latest timeout-minutes: 10 steps: - name: Checkout (full history for tags) @@ -289,6 +289,10 @@ jobs: # still runs. Schedule/dispatch events have no workflow_run context, so the first # clause lets them through. if: github.event_name != 'workflow_run' || github.event.workflow_run.head_branch == github.event.repository.default_branch + # Stock `ubuntu-latest` ON PURPOSE, not the managed pool the rest of CI moved to: + # this job exists to reproduce what an external consumer gets, and the documented + # consumer snippet (docs/tutorials/02-ci-pipeline.md, mirrored by `action-dogfood`) + # says `ubuntu-latest`. A bulk `runs-on:` migration must not sweep this up. runs-on: ubuntu-latest timeout-minutes: 20 steps: diff --git a/CLAUDE.md b/CLAUDE.md index 4794b5f6..2ba409af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -214,7 +214,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033, CE034 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) +When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033, CE034, CE035 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) Adding a user-facing field to one of the models CE030 tracks (`TaskDefinition`, `RunLimits`, `Dataset`, `SimulationConfig` — see `tests/lint/doc_schema_parity.py`) means documenting it in its guide (mention the field name as inline code) or adding an `EXEMPT` entry with a reason it is not user-authored. `make lint` fails otherwise. diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py index 16e3d348..81c6a2c6 100644 --- a/tests/test_verify_published_workflow.py +++ b/tests/test_verify_published_workflow.py @@ -183,7 +183,7 @@ def test_all_three_pin_anchor_readers_agree_on_action_yml(): bump_sed = next( line.strip() for line in _run_body( - _load(RELEASE_WF), "Regenerate uv.lock, bump action.yml pin, and amend release commit" + _load(RELEASE_WF), "Regenerate uv.lock, bump action.yml + plugin.json pins, and amend release commit" ).splitlines() if line.strip().startswith("sed -i -E") and "kept in sync" in line ) From a2a488b067a3078bb3b5c85567e0dc366f40107c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 13 Aug 2026 09:44:11 -0700 Subject: [PATCH 6/9] fix(ci): close the five gate-correctness findings from the code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. `agent_crash` no longer excuses a zero-token run. It is the categorizer's catch-all last resort for any unclassified AgentCrashError, so it is exactly what a missing or broken `claude` CLI produces — arriving under the one label that let the gate exit 0 with an unread ::warning::, disarming its only proof that credential passthrough and the agent runtime work. Split into UNREACHABLE (model/API genuinely unavailable) for the zero-token check and TOLERABLE_AFTER_TOKENS for the harness-error check, where a crash with tokens already billed is a plausible transient. 2. The inline nightly task declares run_limits (max_turns / task_timeout / max_usd). Every RunLimits cap defaults to None, so the repo's only unattended PAID run was bounded solely by the job's timeout-minutes — a cancellation that leaves no run.json for the gate to read, i.e. maximally expensive and minimally diagnosable. A tripped cap produces a row the gate already prints. 3. "Assert PyPI serves this run's artifacts" is now set equality, not containment. Digest-matching each local file said nothing about files we did NOT build, and installers prefer a platform-specific wheel over our py3-none-any — so one planted `...-cp313-manylinux_*.whl` would be what `uv tool install` resolves while every file we built still matched. The point-in-time scope of the guarantee is now stated in both the step comment and the CONTRIBUTING runbook, which claimed more than the check delivered. 4. The promote job's two consumer-visible guards get tests, using the lifted- shell harness this branch already built: the monotonicity check refuses to walk `v0` backwards over a v0.9.5/v0.9.6 git fixture (the 30-day "Re-run failed jobs" hazard) and fails loudly when no vX.Y.Z tag exists; the shape regex rejects `0.9`, `0.9.6rc1` and an injection-shaped value, with the separate `-z` branch owning the empty-version diagnostic. 5. CE035 hardening. The writer scan captured the conversion letter out of a format string (`printf "%s=%s\n"` -> the key `s`), and that non-empty-but- wrong set defeated the "no readable key => skip" contract — a false FAILURE, the one direction the docstring promises the rule can never take. The key must now start at a token boundary. A `needs.` naming a job that does not exist is now a finding rather than a deferral to actionlint, which this repo does not run as a gate. Five previously-unexercised branches gained tests (local-composite arm, writes-no-outputs, nonexistent job, the printf regression, Finding.line); each was mutation-checked to confirm it goes red. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release.yml | 17 ++ .github/workflows/verify-published-action.yml | 38 +++-- CONTRIBUTING.md | 2 +- tests/lint/workflow_outputs.py | 25 ++- tests/test_custom_lint.py | 111 +++++++++++++ tests/test_verify_published_workflow.py | 147 +++++++++++++++++- 6 files changed, 322 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8f9aaeb..a0d5928e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -415,6 +415,11 @@ jobs: # not: the JSON API can lag seconds behind an upload, and a transient must not # redden a publish that actually succeeded (it would also block the re-run story # `skip-existing` exists for). So: mismatch => error, unreachable => warning. + # + # Scope, so the guarantee is not read as wider than it is: this is a POINT-IN-TIME + # set-equality check -- the files PyPI serves for this version at the moment the + # step runs are exactly the files this run built. It cannot see an upload that + # lands after it passes, and it says nothing about any other version. - name: Assert PyPI serves this run's artifacts env: VERSION: ${{ needs.release.outputs.version }} @@ -457,6 +462,18 @@ jobs: bad.append(f"{path.name}: PyPI serves sha256 {got}, this run built {want}") else: print(f" {path.name}: sha256 matches ({want[:12]}...)") + + # Digest-matching every local file proves nothing about files we did NOT build. + # An EXTRA distribution under our exact version is the more dangerous half of + # the threat this step exists for: installers prefer a platform-specific wheel + # over our `py3-none-any`, so a single planted `...-cp313-manylinux_*.whl` would + # be what `uv tool install coder-eval==` (action.yml) actually resolves, + # while every file we built still matches byte-for-byte. Set equality, not + # containment. + extra = sorted(set(remote) - {p.name for p in local}) + if extra: + bad.append(f"PyPI serves {len(extra)} file(s) this run did not build: {', '.join(extra)}") + if bad: print("::error title=Published artifact is not ours::PyPI does not serve the artifacts this run " f"built for {version}: " + "; ".join(bad) + ". Do NOT promote: investigate before moving " diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 8949cabc..a3d33788 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -336,6 +336,18 @@ jobs: # run is isolated and cheap regardless of the runner's state. setting_sources: [] + # This is the repo's only UNATTENDED, PAID run: a nightly cron with a live + # ANTHROPIC_API_KEY and no human watching. Every RunLimits cap defaults to + # None, so without this block the only bound is the job's `timeout-minutes`, + # which cancels the runner and leaves NO run.json for the gate below to read + # -- the failure is both maximally expensive and minimally diagnosable. These + # caps are ~10x what the one-file task needs; tripping one produces a + # COST_BUDGET_EXCEEDED / TIMEOUT row the gate prints and fails on. + run_limits: + max_turns: 5 + task_timeout: 300 + max_usd: 0.25 + success_criteria: - type: "file_exists" path: "hello.txt" @@ -425,12 +437,20 @@ jobs: import json, os, sys, xml.etree.ElementTree as ET # Error categories that mean the failure is UPSTREAM of the published action -- - # the model/API was unavailable or the agent process died on a transient. Those - # must not be reported as broken wiring: this check answers "does the published - # action still work", not "is the model available right now". Everything else - # (auth, billing, config, sandbox/install, or no category at all) IS the - # published action's problem and stays a hard error. - UPSTREAM = {"agent_api_error", "agent_rate_limit", "agent_timeout", "agent_crash"} + # the model/API was unavailable. Those must not be reported as broken wiring: + # this check answers "does the published action still work", not "is the model + # available right now". Everything else (auth, billing, config, sandbox/install, + # or no category at all) IS the published action's problem and stays a hard error. + UNREACHABLE = {"agent_api_error", "agent_rate_limit", "agent_timeout"} + + # `agent_crash` is deliberately NOT in the set above. It is the categorizer's + # catch-all last resort (errors/categorization.py: any AgentCrashError that + # matched no more specific rule), so it is exactly what a MISSING OR BROKEN + # `claude` CLI produces -- the failure this gate exists to catch, arriving under + # the label that would excuse it. It is tolerable only once the run has proven + # the runtime works at all, i.e. after the zero-token check below has passed; + # a crash mid-run with tokens already billed is a plausible transient. + TOLERABLE_AFTER_TOKENS = UNREACHABLE | {"agent_crash"} data = json.load(open(os.environ["RUN_JSON"], encoding="utf-8")) rows = data.get("task_results") or [] @@ -467,9 +487,9 @@ jobs: # passthrough for an Anthropic outage is how a check earns being ignored. So # discriminate on the category run.json already records per row. if tokens <= 0: - if any(c in UPSTREAM for c in categories): + if any(c in UNREACHABLE for c in categories): print("::warning::no tokens consumed, but the run reports an upstream failure " - f"(error_category={[c for c in categories if c in UPSTREAM]}) -- the model/API was " + f"(error_category={[c for c in categories if c in UNREACHABLE]}) -- the model/API was " "unavailable, which says nothing about the published action. Inconclusive, not a " "wiring failure; re-run to confirm.") sys.exit(0) @@ -486,7 +506,7 @@ jobs: # the category says the cause was upstream. harness_errors = [ (s, c) for s, c in zip(statuses, categories, strict=True) - if s in {"ERROR", "BUILD_FAILED"} and c not in UPSTREAM + if s in {"ERROR", "BUILD_FAILED"} and c not in TOLERABLE_AFTER_TOKENS ] if harness_errors: print(f"::error::task(s) failed with a harness/environment error, not a model flake: " diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 82cfe808..c55eff1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,7 +130,7 @@ Actions tab and pick a bump level. It runs three jobs, in this order: | Job | Does | Re-runnable? | |-----|------|--------------| | `release` | bumps the version, bumps `action.yml`'s `version:` pin, tags `vX.Y.Z`, pushes `main` + that tag, builds the wheel/sdist, pushes the GHCR agent image | **No** — re-running bumps and tags a *second* version | -| `publish-pypi` | publishes the wheel/sdist to public PyPI (OIDC, `pypi` environment) and asserts PyPI serves the exact files this run built | Yes | +| `publish-pypi` | publishes the wheel/sdist to public PyPI (OIDC, `pypi` environment) and asserts that, at that moment, the files PyPI serves for this version are exactly the ones this run built — same names, same sha256, nothing extra | Yes | | `promote` | moves the `v0` tag and cuts the GitHub Release | Yes | `v0` is the ref every consumer pins (`uses: UiPath/coder_eval@v0`), and the diff --git a/tests/lint/workflow_outputs.py b/tests/lint/workflow_outputs.py index fb7b0672..7a901286 100644 --- a/tests/lint/workflow_outputs.py +++ b/tests/lint/workflow_outputs.py @@ -48,8 +48,14 @@ NEEDS_OUTPUT_REF = re.compile(r"needs\.(?P[A-Za-z_][A-Za-z0-9_-]*)\.outputs\.(?P[A-Za-z_][A-Za-z0-9_.-]*)") # `echo "key=value"` / `printf 'key=%s' …` / `echo "key<[A-Za-z_][A-Za-z0-9_.-]*)(?:=|<<)""") +# Loose about what precedes the key, but the key must start at a TOKEN boundary — the +# start of the arguments, whitespace, or an opening quote. Without that anchor the lazy +# prefix walks into a format string and reports the conversion letter as the key: +# `printf "%s=%s\n" "$KEY" "$VAL"` yielded `{'s'}`, a non-empty set, which defeats the +# "no readable key => skip the step" contract in `_written_keys` and turns every real +# reference to that step into a false CE035 failure. See the docstring: over-approximating +# writers is safe, INVENTING one is not. +OUTPUT_WRITE = re.compile(r"""(?:echo|printf)\s+(?:[^\n]*?["'\s])?(?P[A-Za-z_][A-Za-z0-9_.-]*)(?:=|<<)""") @dataclass(frozen=True) @@ -200,7 +206,20 @@ def _check_needs_refs( continue seen.add((producer, key)) if producer not in declared: - continue # unknown job name — actionlint's territory, not this rule's + # Fully enumerable from this file — every job name is right here — and + # it fails exactly like an undeclared key: the reference expands to ''. + # Not deferred to actionlint, which this repo does not run as a gate + # (.github/actionlint.yaml says so in its own header). + findings.append( + Finding( + path, + _first_line_containing(lines, match.group(0)), + f"job '{job_name}': `needs.{producer}.outputs.{key}` names job " + f"'{producer}', which does not exist in this workflow " + f"(jobs present: {sorted(declared)}). It expands to the empty string", + ) + ) + continue if key not in declared[producer]: findings.append( Finding( diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index e3f79983..daaa3e0c 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -2853,3 +2853,114 @@ def test_skips_third_party_actions_and_unreadable_writers(self, tmp_path: Path): encoding="utf-8", ) assert find_unresolved_output_refs([wf], tmp_path) == [] + + def test_catches_an_undeclared_local_composite_output(self, tmp_path: Path): + """The `uses: ./` arm: outputs ARE on disk, so a typo against them is enumerable. + Without this, deleting the composite branch leaves the suite green.""" + from tests.lint.workflow_outputs import find_unresolved_output_refs + + (tmp_path / "action.yml").write_text("name: coder_eval\noutputs:\n run-dir:\n value: x\n", encoding="utf-8") + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" + " j:\n" + " steps:\n" + " - id: gate\n" + " uses: ./\n" + " - run: echo ${{ steps.gate.outputs.rundir }} ${{ steps.gate.outputs.run-dir }}\n", + encoding="utf-8", + ) + findings = find_unresolved_output_refs([wf], tmp_path) + assert len(findings) == 1, [str(f) for f in findings] + assert "outputs.rundir`" in findings[0].message + assert "['run-dir']" in findings[0].message, "the message must name the real outputs" + + def test_catches_a_reference_to_a_step_that_writes_no_outputs(self, tmp_path: Path): + """A `run:` step that never touches $GITHUB_OUTPUT writes nothing — distinct from + the 'unreadable writers' skip, and the branch that reports it (`writes no outputs`) + was otherwise unexercised.""" + from tests.lint.workflow_outputs import find_unresolved_output_refs + + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" + " j:\n" + " steps:\n" + " - id: quiet\n" + " run: echo 'this step sets nothing'\n" + " - run: echo ${{ steps.quiet.outputs.anything }}\n", + encoding="utf-8", + ) + findings = find_unresolved_output_refs([wf], tmp_path) + assert len(findings) == 1, [str(f) for f in findings] + assert "writes no outputs" in findings[0].message + + def test_catches_a_needs_reference_to_a_nonexistent_job(self, tmp_path: Path): + """A typo'd job name fails identically to a typo'd key — it expands to '' — and is + just as enumerable, so it is a finding rather than a deferral to actionlint (which + this repo documents as advisory, not a gate).""" + from tests.lint.workflow_outputs import find_unresolved_output_refs + + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" + " release:\n" + " outputs:\n" + " version: ${{ steps.v.outputs.version }}\n" + " steps:\n" + " - id: v\n" + ' run: echo "version=1.0.0" >> "$GITHUB_OUTPUT"\n' + " promote:\n" + " needs: [release]\n" + " if: needs.relase.outputs.version != ''\n" + " steps:\n" + " - run: echo hi\n", + encoding="utf-8", + ) + findings = find_unresolved_output_refs([wf], tmp_path) + assert len(findings) == 1, [str(f) for f in findings] + assert "names job 'relase', which does not exist" in findings[0].message + assert "['promote', 'release']" in findings[0].message + + def test_a_dynamic_printf_writer_is_unreadable_not_a_bogus_key(self, tmp_path: Path): + """Regression: the writer scan used to capture the conversion letter out of a + format string (`printf "%s=%s\\n"` -> the key `s`). That non-empty-but-wrong set + defeats the "no readable key => skip" contract and false-FAILS a correct workflow, + which is the one direction the docstring promises the rule can never take.""" + from tests.lint.workflow_outputs import _written_keys, find_unresolved_output_refs + + assert _written_keys({"run": 'printf "%s=%s\\n" "$K" "$V" >> "$GITHUB_OUTPUT"'}) is None + assert _written_keys({"run": 'echo "pin=$PIN" >> "$GITHUB_OUTPUT"'}) == {"pin"} + + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" + " j:\n" + " steps:\n" + " - id: dyn\n" + ' run: printf "%s=%s\\n" "$K" "$V" >> "$GITHUB_OUTPUT"\n' + " - run: echo ${{ steps.dyn.outputs.pin }}\n", + encoding="utf-8", + ) + assert find_unresolved_output_refs([wf], tmp_path) == [] + + def test_finding_reports_the_line_of_the_offending_reference(self, tmp_path: Path): + """`Finding.line` is what sends a developer to the right place; nothing asserted + it, so it could regress to a constant 1 with a green suite.""" + from tests.lint.workflow_outputs import find_unresolved_output_refs + + wf = tmp_path / "w.yml" + wf.write_text( + "jobs:\n" # 1 + " j:\n" # 2 + " steps:\n" # 3 + " - id: s\n" # 4 + ' run: echo "a=1" >> "$GITHUB_OUTPUT"\n' # 5 + " - run: echo hi\n" # 6 + " - run: echo ${{ steps.s.outputs.b }}\n", # 7 + encoding="utf-8", + ) + findings = find_unresolved_output_refs([wf], tmp_path) + assert len(findings) == 1 + assert findings[0].line == 7, f"expected line 7, got {findings[0].line}" + assert str(findings[0]).startswith(f"{wf}:7 — ") diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py index 81c6a2c6..a771673b 100644 --- a/tests/test_verify_published_workflow.py +++ b/tests/test_verify_published_workflow.py @@ -75,23 +75,47 @@ def _line_after(body: str, anchor: str) -> str: raise AssertionError(f"anchor '{anchor}' not found — did the step get reflowed?") -def _bash(script: str, stdin: str = "", env: dict[str, str] | None = None) -> str: - """Run a snippet lifted verbatim out of a workflow. Inputs go through the - environment, never argv or interpolation, so a fixture value carrying quotes cannot - be mistaken for shell syntax.""" - proc = subprocess.run( +def _bash_result( + script: str, + stdin: str = "", + env: dict[str, str] | None = None, + cwd: Path | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a lifted snippet and return the raw result, exit code included. + + Used for the guards, whose whole contract is *refusing* — asserting exit 0 would + make every one of them untestable. + """ + return subprocess.run( ["bash", "-c", script], input=stdin, capture_output=True, text=True, encoding="utf-8", env={**os.environ, **(env or {})}, + cwd=cwd, check=False, ) + + +def _bash(script: str, stdin: str = "", env: dict[str, str] | None = None) -> str: + """Run a snippet lifted verbatim out of a workflow. Inputs go through the + environment, never argv or interpolation, so a fixture value carrying quotes cannot + be mistaken for shell syntax.""" + proc = _bash_result(script, stdin=stdin, env=env) assert proc.returncode == 0, f"script failed ({proc.returncode}): {proc.stderr}" return proc.stdout +def _body_before(body: str, marker: str) -> str: + """The lines of a ``run:`` body up to (excluding) the first line starting with + ``marker``. Lets a guard be exercised without the irreversible action it guards.""" + lines = body.splitlines() + cut = next((i for i, line in enumerate(lines) if line.strip().startswith(marker)), None) + assert cut is not None, f"marker {marker!r} not found — did the step get restructured?" + return "\n".join(lines[:cut]) + + def _slug_pipeline() -> str: """The one-line slug derivation in the preflight job, lifted from its anchor.""" body = _run_body(_load(VERIFY_WF), "Verify Marketplace listing resolves") @@ -215,3 +239,116 @@ def test_inline_consumer_task_yaml_loads(tmp_path: Path): assert task.task_id == "published_action_smoke" assert task.success_criteria, "the nightly's task must assert something" + + +def test_inline_consumer_task_declares_run_limits(tmp_path: Path): + """The nightly is unattended and paid: an uncapped run burns spend until the job's + `timeout-minutes` cancels the runner, which produces no run.json for the gate to + read. Every RunLimits cap defaults to None, so omitting the block is silent.""" + from coder_eval.orchestration.task_loader import load_task + + body = _run_body(_load(VERIFY_WF), "Write a consumer task YAML") + lines = body.splitlines() + start = next(i for i, line in enumerate(lines) if "<<'YAML'" in line) + end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "YAML") + path = tmp_path / "published_smoke.yaml" + path.write_text("\n".join(lines[start + 1 : end]) + "\n", encoding="utf-8") + + limits = load_task(path)[0].run_limits + assert limits is not None, "the unattended paid task must declare run_limits" + assert limits.max_turns, "an unbounded turn count on a cron-triggered paid run" + assert limits.max_usd, "an unbounded spend on a cron-triggered paid run" + assert limits.task_timeout, "no wall-clock cap below the job's timeout-minutes" + + +# -------------------------------------------------------------------------------------- +# 5. the promote job's two consumer-visible guards +# +# `promote` moves `v0` — the ref every `uses: UiPath/coder_eval@v0` consumer resolves. +# Its two guards are the branch's highest-stakes new shell and cannot be exercised +# before merge (promote only runs on a real release), so they are lifted and run here. +# -------------------------------------------------------------------------------------- + + +def _version_shape_guard() -> str: + return _run_body(_load(RELEASE_WF), "Validate version shape") + + +@pytest.mark.parametrize("version", ["0.9.6", "1.0.0", "10.20.30"]) +def test_version_shape_guard_accepts_real_releases(version: str): + proc = _bash_result(_version_shape_guard(), env={"VERSION": version}) + assert proc.returncode == 0, f"guard rejected the valid version {version!r}: {proc.stdout}{proc.stderr}" + assert f"promoting v{version}" in proc.stdout + + +@pytest.mark.parametrize( + "version", + [ + "0.9", # too few components + "0.9.6.1", # too many + "0.9.6rc1", # a prerelease is not promotable to the major tag + "v0.9.6", # already prefixed; would yield `vv0.9.6` in the ref + "0.9.6 && echo pwned", # shape check is defence-in-depth for the `ref:` interpolation + ], +) +def test_version_shape_guard_refuses_malformed_versions(version: str): + proc = _bash_result(_version_shape_guard(), env={"VERSION": version}) + assert proc.returncode == 1, f"guard accepted the malformed version {version!r}" + assert "malformed version" in proc.stdout + + +def test_version_shape_guard_owns_the_empty_version_case(): + """The job's `if:` deliberately does not gate on the output being non-empty (that + shape resolves to SKIPPED-green and strands a release), so this guard is the sole + enforcement point — and it must say so, not just fail the regex.""" + proc = _bash_result(_version_shape_guard(), env={"VERSION": ""}) + assert proc.returncode == 1 + assert "Release version unavailable" in proc.stdout, ( + "an empty version must get its own partial-re-run diagnostic, not the generic " + f"malformed-version message: {proc.stdout}" + ) + + +def _tag_repo(root: Path, tags: tuple[str, ...]) -> Path: + """A git repo carrying `tags` on one commit, for the monotonicity guard to read.""" + run = ["git", "-c", "user.email=t@t", "-c", "user.name=t"] + subprocess.run(["git", "init", "-q", "-b", "main", str(root)], check=True) + (root / "f.txt").write_text("x", encoding="utf-8") + subprocess.run([*run, "add", "f.txt"], cwd=root, check=True, capture_output=True) + subprocess.run([*run, "commit", "-qm", "c"], cwd=root, check=True, capture_output=True) + for tag in tags: + subprocess.run(["git", "tag", tag], cwd=root, check=True, capture_output=True) + return root + + +def _monotonicity_guard() -> str: + """The guard only — everything before the irreversible `git tag -f` / `git push -f`.""" + body = _run_body(_load(RELEASE_WF), "Move major action tag (vN -> this release)") + guard = _body_before(body, "git tag -f") + assert "NEWEST" in guard, "the monotonicity check is no longer above the tag move" + return guard + + +def test_monotonicity_guard_allows_promoting_the_newest_release(tmp_path: Path): + repo = _tag_repo(tmp_path / "newest", ("v0.9.5", "v0.9.6", "v0")) + proc = _bash_result(_monotonicity_guard(), env={"VERSION": "0.9.6"}, cwd=repo) + assert proc.returncode == 0, f"{proc.stdout}{proc.stderr}" + + +def test_monotonicity_guard_refuses_to_walk_the_major_tag_backwards(tmp_path: Path): + """The 30-day "Re-run failed jobs" hazard: replaying an OLD release's promote after a + newer one shipped would downgrade every consumer pinned to the major tag.""" + repo = _tag_repo(tmp_path / "backwards", ("v0.9.5", "v0.9.6", "v0")) + proc = _bash_result(_monotonicity_guard(), env={"VERSION": "0.9.5"}, cwd=repo) + assert proc.returncode == 1, "the guard promoted a superseded release" + assert "backwards" in proc.stdout + assert "v0.9.6" in proc.stdout, "the operator is not told which release to promote instead" + + +def test_monotonicity_guard_refuses_when_no_release_tag_exists(tmp_path: Path): + """`v0` alone must not satisfy the newest-tag lookup: the grep keeps only vX.Y.Z, and + an empty result has to fail loudly rather than compare against the empty string.""" + repo = _tag_repo(tmp_path / "majoronly", ("v0",)) + proc = _bash_result(_monotonicity_guard(), env={"VERSION": "0.9.6"}, cwd=repo) + assert proc.returncode == 1 + assert "no vX.Y.Z release tag found" in proc.stdout From c182890cfcdae1c43ba68ea40db03fc4d3aaeee9 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 14 Aug 2026 11:49:08 -0700 Subject: [PATCH 7/9] fix(ci): fail the published-action gate on a run-limit breach Reviewer gap: TIMEOUT, TOKEN_BUDGET_EXCEEDED and COST_BUDGET_EXCEEDED map to the "failed" reporting category, not "error", so they fell through the ERROR/BUILD_FAILED branch AND the exit-contract check (which only demands green when every status is SUCCESS). A tripped cost cap on the repo's one unattended paid run therefore burned the budget and left the job GREEN -- the exact failure those caps were added to make visible. The task YAML's own comment already claimed the gate failed on such a row; now it does. MAX_TURNS_EXHAUSTED stays tolerated: it is the model-quality outcome, like an unmet criterion. The branch is deliberately not excused by error_category -- a task timeout is always categorized agent_timeout, so excusing the UNREACHABLE set would make it dead code on its likeliest member. Bound by two tests that lift the status literals out of the gate's Python block via AST: one mirrors the explicit-mapping guard in models/enums.py (every FinalStatus must be hard-failed or deliberately tolerated, so a new member cannot default to silent-green), the other ties each declared cap to the status that catches its breach. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/verify-published-action.yml | 33 +++++++- tests/test_verify_published_workflow.py | 84 +++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index a3d33788..d625fa40 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -341,8 +341,11 @@ jobs: # None, so without this block the only bound is the job's `timeout-minutes`, # which cancels the runner and leaves NO run.json for the gate below to read # -- the failure is both maximally expensive and minimally diagnosable. These - # caps are ~10x what the one-file task needs; tripping one produces a - # COST_BUDGET_EXCEEDED / TIMEOUT row the gate prints and fails on. + # caps are ~10x what the one-file task needs; tripping the spend or wall-clock + # one produces a COST_BUDGET_EXCEEDED / TIMEOUT row that the gate's run-limit + # branch prints and fails on (both statuses report as "failed", so nothing else + # in the gate would). MAX_TURNS_EXHAUSTED is the one deliberate exception: it is + # the classic model-quality outcome, tolerated like an unmet criterion. run_limits: max_turns: 5 task_timeout: 300 @@ -513,6 +516,32 @@ jobs: f"{harness_errors} -- the published action's install, sandbox, or agent wiring is broken") sys.exit(1) + # Run-limit breaches. models/enums.py maps TIMEOUT and both budget statuses to + # the "failed" reporting category -- the same bucket as an ordinary criterion + # FAILURE -- so without this branch a tripped cap falls through every check + # above AND the exit-contract check below (which only demands green when every + # status is SUCCESS), and the repo's one unattended, paid run goes silently + # GREEN after burning its cost cap. That is the exact failure the caps were + # added to make visible. + # + # These are not "the model did poorly": the caps in the task YAML are ~10x what + # writing one file needs, so reaching one means the run went off the rails -- a + # looping agent, a hung turn, an action feeding a different task. Deliberately + # NOT excused by error_category, unlike the zero-token symptom above: a + # task-level timeout is always categorized agent_timeout (TaskTimeoutError is an + # EvaluationTimeoutError, which errors/categorization.py maps typed and first), + # so excusing the UNREACHABLE set here would make this branch dead code on its + # most likely member. An outage announces itself as an API/rate-limit row long + # before it spends 25 cents; and a red here is re-runnable, whereas a green here + # is undetectable. + RUN_LIMIT_BREACH = {"TIMEOUT", "TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED"} + breaches = [(s, c) for s, c in zip(statuses, categories, strict=True) if s in RUN_LIMIT_BREACH] + if breaches: + print(f"::error::task(s) tripped a run limit: {breaches} -- the caps in the task YAML are " + "~10x what this one-file task needs, so a healthy run cannot reach one. Read the " + "uploaded run dir before re-running; this is an unattended paid job.") + sys.exit(1) + # Exit-contract check, conditional on the model having actually performed. # Ignoring the step's exit code entirely (see the continue-on-error rationale # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py index a771673b..abc81fd1 100644 --- a/tests/test_verify_published_workflow.py +++ b/tests/test_verify_published_workflow.py @@ -352,3 +352,87 @@ def test_monotonicity_guard_refuses_when_no_release_tag_exists(tmp_path: Path): proc = _bash_result(_monotonicity_guard(), env={"VERSION": "0.9.6"}, cwd=repo) assert proc.returncode == 1 assert "no vX.Y.Z release tag found" in proc.stdout + + +# -------------------------------------------------------------------------------------- +# 6. the mechanics gate reasons over FinalStatus values as bare strings +# +# The gate's whole job is deciding which run.json `status` values mean "the published +# action is broken". It spells them as string literals inside a heredoc, so a renamed +# member, or a NEW member nobody classified here, is invisible to every other gate — +# the check would just stop matching and the paid nightly would go green. +# -------------------------------------------------------------------------------------- + + +def _gate_python() -> str: + """The embedded Python block of the `Verify action mechanics` step.""" + body = _run_body(_load(VERIFY_WF), "Verify action mechanics") + lines = body.splitlines() + start = next(i for i, line in enumerate(lines) if "<<'PY'" in line) + end = next(i for i in range(start + 1, len(lines)) if lines[i].strip() == "PY") + return "\n".join(lines[start + 1 : end]) + "\n" + + +def _statuses_the_gate_names() -> set[str]: + """Every status literal the gate compares a run.json `status` against. + + Read out of the AST rather than by regex, so reflowing the block cannot quietly + empty the set and make the coverage assertion below vacuous. + """ + import ast + + tree = ast.parse(_gate_python()) + named: set[str] = set() + for node in ast.walk(tree): + # `s in {...}` / `s == "..."`, plus the RUN_LIMIT_BREACH set the branch is keyed on. + if isinstance(node, ast.Compare) and isinstance(node.left, ast.Name) and node.left.id == "s": + for comparator in node.comparators: + if isinstance(comparator, ast.Set): + named |= {e.value for e in comparator.elts if isinstance(e, ast.Constant)} + elif isinstance(comparator, ast.Constant) and isinstance(comparator.value, str): + named.add(comparator.value) + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Set): + targets = {t.id for t in node.targets if isinstance(t, ast.Name)} + if "RUN_LIMIT_BREACH" in targets: + named |= {e.value for e in node.value.elts if isinstance(e, ast.Constant)} + assert named, "no status literals found — did the gate's Python block get restructured?" + return named + + +def test_gate_classifies_every_final_status(): + """Mirrors the explicit-mapping guard in models/enums.py: a newly added FinalStatus + must be classified here rather than silently defaulting to tolerated. Only the two + "the model did poorly" outcomes are deliberately unnamed — everything else is either + a hard failure or the success the exit-contract check is keyed on.""" + from coder_eval.models import FinalStatus + + named = _statuses_the_gate_names() + unknown = named - {s.value for s in FinalStatus} + assert not unknown, f"the gate compares `status` against non-FinalStatus values {unknown} — dead branches" + + tolerated = {s.value for s in FinalStatus} - named + assert tolerated == {FinalStatus.FAILURE.value, FinalStatus.MAX_TURNS_EXHAUSTED.value}, ( + f"the gate does not classify {tolerated}. Every FinalStatus must be either hard-failed " + "or deliberately tolerated as a model-quality outcome; an unclassified one falls through " + "to a GREEN unattended paid run." + ) + + +def test_gate_hard_fails_the_run_limit_breaches_the_task_declares(): + """The task YAML's caps are only as good as the gate reading their breach status. + + Both budget statuses and TIMEOUT map to the "failed" reporting category — the same + bucket as an ordinary criterion FAILURE — so they are NOT covered by the + ERROR/BUILD_FAILED branch, and nothing else in the gate rejects them. + """ + from coder_eval.models import FinalStatus + + named = _statuses_the_gate_names() + for status in (FinalStatus.TIMEOUT, FinalStatus.TOKEN_BUDGET_EXCEEDED, FinalStatus.COST_BUDGET_EXCEEDED): + assert status.category == "failed", ( + f"{status.value} is no longer categorized 'failed'; re-check whether the gate's " + "run-limit branch is still the thing that catches it" + ) + assert status.value in named, ( + f"a tripped {status.value} cap is not rejected by the gate: the run burns its budget and the job goes green" + ) From 744b66819aece6693ee6ca6b6b3a7205509470a0 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 14 Aug 2026 13:04:20 -0700 Subject: [PATCH 8/9] chore: renumber a rebase-collided lint-rule candidate The antigravity review block landing on main proposed a sleep-loop rule as "CE035"; this branch shipped CE035 as the workflow-outputs resolver. Point the candidate at the next free number so the registry has one meaning per id. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index b93c818d..e067e9ad 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -384,7 +384,8 @@ with the two `action.yml` items above — one considered change to the action's `timeout is None`. Caught independently by two reviewers (`bai-uipath`, `uipreliga`) on the PR, both citing the exact same arithmetic mismatch. **Not promoted in this pass**, but a stronger candidate than most entries here: `uipreliga` proposed a - generic whole-tree rule (their CE035) — for every sleep-loop under + generic whole-tree rule (proposed as CE035, renumbered CE042 here — CE035 shipped as + the workflow-outputs resolver on the published-action branch) — for every sleep-loop under `src/coder_eval/agents/**`, assert its own cycle-count × interval either references a timeout-derived name or is provably below `experiments/default.yaml`'s baseline — that would catch this class of bug in ANY agent, not just this one (confirmed zero From b575f0bb5e9b6a865af9d1b4f4e3de992f558937 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 14 Aug 2026 13:12:41 -0700 Subject: [PATCH 9/9] fix(test): resolve bash by absolute path so Windows CI stops hitting WSL The Windows Smoke Test job went red on every lifted-snippet test in this file. Cause is not the snippets: `subprocess.run(["bash", ...])` resolves a bare name through CreateProcess, which searches System32 BEFORE PATH, and C:\Windows\System32\bash.exe is the WSL launcher stub -- with no distro installed it prints "Windows Subsystem for Linux has no installed distributions" in UTF-16 (so it arrives NUL-interleaved in stdout) and exits 1, which read as "the guard rejected a valid version". Resolve with shutil.which() and invoke the absolute path, which is why tests/test_litellm_config.py's bash check already passes on that same runner: PATH order finds the runner's Git Bash. Falls back to the two known Git Bash locations if PATH happens to list System32 first, and skips -- not fails -- when the host has no POSIX bash at all, since these snippets only ever run on ubuntu runners. The skip sits in the shared helper so a test added later cannot forget it. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_verify_published_workflow.py | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/test_verify_published_workflow.py b/tests/test_verify_published_workflow.py index abc81fd1..d3f25b4c 100644 --- a/tests/test_verify_published_workflow.py +++ b/tests/test_verify_published_workflow.py @@ -25,6 +25,7 @@ from __future__ import annotations import os +import shutil import subprocess from pathlib import Path from typing import Any @@ -42,6 +43,38 @@ ACTION_YML = REPO_ROOT / "action.yml" +def _resolve_bash() -> str | None: + """An ABSOLUTE path to a POSIX bash, or None if the host has none. + + Never the bare name ``"bash"``: on Windows, ``subprocess`` resolves it through + CreateProcess, which searches ``System32`` *before* ``PATH`` — and + ``C:\\Windows\\System32\\bash.exe`` is the WSL launcher stub. With no distro + installed it prints "Windows Subsystem for Linux has no installed distributions" + (in UTF-16, so it arrives NUL-interleaved) and exits 1, which makes every lifted + snippet below fail for a reason that has nothing to do with the snippet. + ``shutil.which`` follows PATH order instead, which is how the Windows runner's Git + Bash is found — the same resolution ``tests/test_litellm_config.py`` already relies + on. The System32 fallback covers a PATH that happens to list it first. + """ + found = shutil.which("bash") + if found and "system32" in found.replace("\\", "/").lower(): + found = next( + ( + candidate + for candidate in ( + r"C:\Program Files\Git\bin\bash.exe", + r"C:\Program Files\Git\usr\bin\bash.exe", + ) + if Path(candidate).exists() + ), + None, + ) + return found + + +BASH = _resolve_bash() + + def _load(path: Path) -> dict[str, Any]: data = yaml.safe_load(path.read_text(encoding="utf-8")) assert isinstance(data, dict), f"{path} did not parse as a mapping" @@ -85,9 +118,16 @@ def _bash_result( Used for the guards, whose whole contract is *refusing* — asserting exit 0 would make every one of them untestable. + + Skips (rather than fails) on a host with no POSIX bash: these snippets are lifted + out of workflows that only ever run on ubuntu runners, so a shell-less host proves + nothing about them. The skip lives here rather than on ~10 marks so a test added + later cannot forget it. """ + if BASH is None: + pytest.skip("no POSIX bash on this host; the lifted workflow snippets need one") return subprocess.run( - ["bash", "-c", script], + [BASH, "-c", script], input=stdin, capture_output=True, text=True,