From e1f1eee2265185ad3ab3d584d49762cccd0ff95a Mon Sep 17 00:00:00 2001 From: Joel Teply Date: Tue, 10 Nov 2026 17:50:55 -0600 Subject: [PATCH] Re-admit latest canary from verified publication completion --- .github/workflows/core-binaries-hourly.yml | 11 ++- .github/workflows/core-binaries.yml | 29 ++++++++ docs/operations/HOURLY-CORE-PUBLICATION.md | 13 +++- tools/scripts/core-binaries-hourly.mjs | 47 ++++++++++--- .../tests/core-binaries-hourly.test.mjs | 68 ++++++++++++++++++- 5 files changed, 149 insertions(+), 19 deletions(-) diff --git a/.github/workflows/core-binaries-hourly.yml b/.github/workflows/core-binaries-hourly.yml index 74a98792cf..8b208eae70 100644 --- a/.github/workflows/core-binaries-hourly.yml +++ b/.github/workflows/core-binaries-hourly.yml @@ -3,7 +3,7 @@ name: core-binaries-hourly on: workflow_run: - workflows: ['Continuum Rust Tests', 'core-binaries'] + workflows: ['Continuum Rust Tests'] types: [completed] branches: [canary] schedule: @@ -17,6 +17,7 @@ on: pull_request: paths: - '.github/workflows/core-binaries-hourly.yml' + - '.github/workflows/core-binaries.yml' - 'tools/scripts/core-binaries-hourly.mjs' - 'tools/scripts/tests/core-binaries-hourly.test.mjs' - 'tools/scripts/publication-build-key.mjs' @@ -38,7 +39,7 @@ jobs: - run: node --test tools/scripts/tests/core-binaries-hourly.test.mjs tools/scripts/tests/publication-build-key.test.mjs dispatch: needs: fixtures - # Successful checks and completed publication wake the same admission as cron. + # Successful checks wake the same admission as cron; publisher completion calls it directly. # Failed publication does not create an immediate automatic retry loop. if: >- github.event_name == 'workflow_dispatch' || @@ -48,10 +49,8 @@ jobs: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'canary' && github.event.workflow_run.head_repository.full_name == github.repository && - ((github.event.workflow_run.name == 'Continuum Rust Tests' && - github.event.workflow_run.event == 'push') || - (github.event.workflow_run.name == 'core-binaries' && - github.event.workflow_run.event == 'workflow_dispatch'))))) + github.event.workflow_run.name == 'Continuum Rust Tests' && + github.event.workflow_run.event == 'push'))) runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/core-binaries.yml b/.github/workflows/core-binaries.yml index 0961b62bbb..50480b2b76 100644 --- a/.github/workflows/core-binaries.yml +++ b/.github/workflows/core-binaries.yml @@ -580,3 +580,32 @@ jobs: echo "pruning $tag" gh release delete "$tag" -R "$GITHUB_REPOSITORY" --yes --cleanup-tag done + + admit-next: + name: admit latest canary after publication + needs: [build, publish-check, publish-script-test, prune] + if: >- + success() && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/canary' && + vars.CORE_BINARIES_HOURLY_ENABLED == 'true' + runs-on: ubuntu-latest + permissions: + contents: read + actions: write + # Same admission mutex as cron/check completion. The next build remains queued + # behind this workflow's existing canary concurrency group until we finish. + concurrency: + group: core-binaries-hourly-dispatch + cancel-in-progress: false + steps: + - uses: actions/checkout@v4 + with: + ref: canary + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: node tools/scripts/core-binaries-hourly.mjs + env: + GH_TOKEN: ${{ github.token }} + DRY_RUN: 'false' + PUBLICATION_COMPLETION_KEY: ${{ inputs.build_key }} diff --git a/docs/operations/HOURLY-CORE-PUBLICATION.md b/docs/operations/HOURLY-CORE-PUBLICATION.md index 0ccb070811..c2232e3a67 100644 --- a/docs/operations/HOURLY-CORE-PUBLICATION.md +++ b/docs/operations/HOURLY-CORE-PUBLICATION.md @@ -3,7 +3,14 @@ The enabled dispatcher runs after successful `Continuum Rust Tests` push runs on canary, after successful `core-binaries` publication dispatches, and hourly, for the latest unpublished canary revision. These events use the same admission script -and concurrency group. Publication completion rechecks the latest key when its +and concurrency group. The successful publisher calls admission directly from its +final job, after build, verification, script tests and pruning succeed. It verifies +the current run identity/attempt and complete exact-key manifests before excluding +only that finishing run from the active-publication check. Every other active run +still blocks. The next build queues behind the existing non-cancelling canary +publication group until this publisher exits. This replaces the publication +`workflow_run` trigger; successful Rust checks retain their trigger. +Publication completion rechecks the latest key when its earlier wake skipped behind an active build; complete manifests make that recheck a no-op. PR/fork runs, failures and manually started test runs do not trigger publication. Failed publication waits for the next ordinary wake rather than @@ -39,7 +46,9 @@ No service or deployment ledger changes are involved. The dispatcher checks all three platforms' manifest, archive and checksum assets, including full manifest git SHA, before declaring a revision published. Missing or partial publication is eligible; API errors fail visibly instead of launching a -build on a guess. Any queued/running publication prevents another dispatch. +build on a guess. Any queued/running publication prevents another dispatch, except +the verified finishing publisher performing its own completion handoff. An incomplete +finishing release refuses handoff rather than creating a retry loop. Manual `core-binaries` dispatch remains available independently for urgent repair. It requires a full `build_key` selected from canary history. The dispatcher and consumer share `core/continuum-cli-lifecycle/build-inputs.txt`: the newest commit diff --git a/tools/scripts/core-binaries-hourly.mjs b/tools/scripts/core-binaries-hourly.mjs index 6792dcb103..5e80a221e3 100644 --- a/tools/scripts/core-binaries-hourly.mjs +++ b/tools/scripts/core-binaries-hourly.mjs @@ -4,11 +4,8 @@ import { resolveBuildKey } from './publication-build-key.mjs'; export const platforms = ['macos-arm64', 'macos-x86_64', 'windows-x86_64']; -export async function planPublication(api, repo, resolve = resolveBuildKey) { +async function publicationComplete(api, repo, sha) { const prefix = `/repos/${repo}`; - const { sha: tip } = await api(`${prefix}/commits/canary`); - if (!/^[0-9a-f]{40}$/.test(tip)) throw new Error('canary did not resolve to a full SHA'); - const sha = await resolve(tip, process.cwd()); const tag = `canary-${sha.slice(0, 12)}`; const release = await api(`${prefix}/releases/tags/${tag}`, { allow404: true }); if (release !== null && !Array.isArray(release.assets)) throw new Error('release assets response is malformed'); @@ -29,17 +26,45 @@ export async function planPublication(api, repo, resolve = resolveBuildKey) { } } } - if (complete) return { sha, action: 'skip', reason: 'all platform manifests published' }; + return complete; +} + +// Only the successful publisher may hand off its own finishing run. The release +// manifests prove publication, while the run receipt binds the exclusion to this +// repository, workflow, branch and attempt. No arbitrary active-run exemption. +async function finishingPublication(api, repo, completion) { + const { runId, attempt, key } = completion; + if (!Number.isSafeInteger(runId) || runId <= 0 || + !Number.isSafeInteger(attempt) || attempt <= 0 || !/^[0-9a-f]{40}$/.test(key)) { + throw new Error('malformed publication completion'); + } + const run = await api(`/repos/${repo}/actions/runs/${runId}`); + if (run.id !== runId || run.run_attempt !== attempt || run.event !== 'workflow_dispatch' || + run.path !== '.github/workflows/core-binaries.yml' || run.head_branch !== 'canary' || + run.repository?.full_name !== repo || run.head_repository?.full_name !== repo || + run.status !== 'in_progress') throw new Error('publication completion run does not match'); + if (!await publicationComplete(api, repo, key)) throw new Error('finishing publication is incomplete'); + return runId; +} + +export async function planPublication(api, repo, resolve = resolveBuildKey, completion = null) { + const finishingId = completion === null ? null : await finishingPublication(api, repo, completion); + const prefix = `/repos/${repo}`; + const { sha: tip } = await api(`${prefix}/commits/canary`); + if (!/^[0-9a-f]{40}$/.test(tip)) throw new Error('canary did not resolve to a full SHA'); + const sha = await resolve(tip, process.cwd()); + if (await publicationComplete(api, repo, sha)) return { sha, action: 'skip', reason: 'all platform manifests published' }; const runs = await api(`${prefix}/actions/workflows/core-binaries.yml/runs?event=workflow_dispatch&per_page=100`); if (!Array.isArray(runs.workflow_runs)) throw new Error('workflow runs response is malformed'); - if (runs.workflow_runs.some(run => run.event === 'workflow_dispatch' && run.status !== 'completed')) { + if (runs.workflow_runs.some(run => run.event === 'workflow_dispatch' && run.status !== 'completed' && + (finishingId === null || run.id !== finishingId))) { return { sha, action: 'skip', reason: 'publication already queued or running' }; } return { sha, action: 'dispatch', reason: 'latest canary is not fully published' }; } -export async function runPublication(api, repo, dryRun, resolve = resolveBuildKey) { - const plan = await planPublication(api, repo, resolve); +export async function runPublication(api, repo, dryRun, resolve = resolveBuildKey, completion = null) { + const plan = await planPublication(api, repo, resolve, completion); if (plan.action === 'dispatch' && !dryRun) { // Keep the canary concurrency group while pinning the build's immutable identity. await api(`/repos/${repo}/actions/workflows/core-binaries.yml/dispatches`, { @@ -64,5 +89,9 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (!response.ok) throw new Error(`GitHub ${method} ${path}: HTTP ${response.status}`); return response.status === 204 ? null : response.json(); }; - console.log(JSON.stringify(await runPublication(api, repo, process.env.DRY_RUN === 'true'))); + const completion = process.env.PUBLICATION_COMPLETION_KEY === undefined ? null : { + runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT), + key: process.env.PUBLICATION_COMPLETION_KEY, + }; + console.log(JSON.stringify(await runPublication(api, repo, process.env.DRY_RUN === 'true', resolveBuildKey, completion))); } diff --git a/tools/scripts/tests/core-binaries-hourly.test.mjs b/tools/scripts/tests/core-binaries-hourly.test.mjs index d0f179b93a..16b4490c03 100644 --- a/tools/scripts/tests/core-binaries-hourly.test.mjs +++ b/tools/scripts/tests/core-binaries-hourly.test.mjs @@ -87,11 +87,15 @@ test('cron, successful canary checks and publication completion share admission const workflow = await readFile(new URL('../../../.github/workflows/core-binaries-hourly.yml', import.meta.url), 'utf8'); const publisher = await readFile(new URL('../../../.github/workflows/core-binaries.yml', import.meta.url), 'utf8'); assert.match(workflow, /cron: '23 \* \* \* \*'/); - assert.match(workflow, /workflow_run:\r?\n workflows: \['Continuum Rust Tests', 'core-binaries'\]\r?\n types: \[completed\]\r?\n branches: \[canary\]/); + assert.match(workflow, /workflow_run:\r?\n workflows: \['Continuum Rust Tests'\]\r?\n types: \[completed\]\r?\n branches: \[canary\]/); assert.match(workflow, /workflow_run\.conclusion == 'success'/); assert.match(workflow, /workflow_run\.event == 'push'/); assert.match(workflow, /workflow_run\.name == 'Continuum Rust Tests' &&\s+github\.event\.workflow_run\.event == 'push'/); - assert.match(workflow, /workflow_run\.name == 'core-binaries' &&\s+github\.event\.workflow_run\.event == 'workflow_dispatch'/); + assert.doesNotMatch(workflow, /workflow_run\.name == 'core-binaries'/); + assert.match(publisher, /needs: \[build, publish-check, publish-script-test, prune\]/); + assert.match(publisher, /success\(\) && github.event_name == 'workflow_dispatch'/); + assert.match(publisher, /group: core-binaries-hourly-dispatch/); + assert.match(publisher, /PUBLICATION_COMPLETION_KEY: \$\{\{ inputs.build_key \}\}/); assert.match(workflow, /workflow_run\.head_branch == 'canary'/); assert.match(workflow, /workflow_run\.head_repository\.full_name == github\.repository/); @@ -106,3 +110,63 @@ test('cron, successful canary checks and publication completion share admission assert.match(publisher, /GITHUB_SHA="\$BUILD_SHA" bash/); assert.match(publisher, /GITHUB_SHA="\$BUILD_KEY" bash/); }); + +const oldKey = 'b'.repeat(40); +const completion = { runId: 17, attempt: 2, key: oldKey }; +function completing({ runPatch = {}, partial = false, sameKey = false, others = [], failReceipt = false, badManifest = false } = {}) { + const live = source(); + const published = source({ complete: true }); + const calls = []; + const self = { id: 17, run_attempt: 2, event: 'workflow_dispatch', status: 'in_progress', + path: '.github/workflows/core-binaries.yml', head_branch: 'canary', + repository: { full_name: 'Example/repo' }, head_repository: { full_name: 'Example/repo' }, ...runPatch }; + const api = async (path, options = {}) => { + calls.push({ path, options }); + if (path.endsWith('/actions/runs/17')) { + if (failReceipt) throw new Error('receipt unavailable'); + return self; + } + if (path.includes('/runs?')) return { workflow_runs: [self, ...others] }; + if (path.endsWith(`/releases/tags/canary-${oldKey.slice(0, 12)}`)) { + return partial ? null : published.api(path, options); + } + if (path.includes('/releases/assets/')) { + const manifest = await published.api(path, options); + return { ...manifest, git_sha: badManifest ? sha : oldKey }; + } + return live.api(path, options); + }; + return { calls, execute: (receipt = completion) => + runPublication(api, 'Example/repo', false, async () => sameKey ? oldKey : sha, receipt) }; +} + +test('publisher handoff admits a newer key once, and stops at a complete same key', async () => { + const next = completing(); + assert.equal((await next.execute()).dispatched, true); + assert.equal(next.calls.filter(c => c.options.method === 'POST').length, 1); + assert.deepEqual(next.calls.find(c => c.options.method === 'POST').options.body, + { ref: 'canary', inputs: { build_key: sha } }); + assert.equal((await completing({ sameKey: true }).execute()).dispatched, false); + for (const status of ['queued', 'in_progress', 'waiting']) { + const other = completing({ others: [{ id: 18, event: 'workflow_dispatch', status }] }); + assert.equal((await other.execute()).dispatched, false); + assert.equal(other.calls.filter(c => c.options.method === 'POST').length, 0); + } +}); + +test('handoff refuses partial publication, stale attempt, foreign run and uncertain receipt', async () => { + for (const options of [{ partial: true }, { failReceipt: true }, { badManifest: true }, + ...[{ id: 18 }, { run_attempt: 1 }, { event: 'pull_request' }, { status: 'completed' }, + { path: '.github/workflows/unrelated.yml' }, { head_branch: 'feature' }, + { repository: { full_name: 'Foreign/repo' } }, { head_repository: null }].map(runPatch => ({ runPatch }))]) { + const invalid = completing(options); + await assert.rejects(invalid.execute()); + assert.equal(invalid.calls.filter(c => c.options.method === 'POST').length, 0); + } + for (const receipt of [{ ...completion, runId: NaN }, { ...completion, attempt: 0 }, + { ...completion, key: 'short' }]) { + const invalid = completing(); + await assert.rejects(invalid.execute(receipt), /malformed/); + assert.equal(invalid.calls.length, 0); + } +});