From 761fbc8bb80f5f2102871b14214119af9e520234 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 10 Nov 2026 16:59:12 +0000 Subject: [PATCH 1/6] fix(ci): group runner patch failures in one tracker --- .github/scripts/report-workflow-failure.cjs | 86 +++++++ .../scripts/report-workflow-failure.test.cjs | 221 ++++++++++++++++++ .../workflows/create-issue-on-failure.yaml | 44 ++-- .github/workflows/test-failure-reporting.yaml | 29 +++ .../plans/2026-11-10-runner-patch-tracker.md | 58 +++++ .../2026-11-10-runner-patch-tracker-design.md | 29 +++ 6 files changed, 437 insertions(+), 30 deletions(-) create mode 100644 .github/scripts/report-workflow-failure.cjs create mode 100644 .github/scripts/report-workflow-failure.test.cjs create mode 100644 .github/workflows/test-failure-reporting.yaml create mode 100644 docs/superpowers/plans/2026-11-10-runner-patch-tracker.md create mode 100644 docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md diff --git a/.github/scripts/report-workflow-failure.cjs b/.github/scripts/report-workflow-failure.cjs new file mode 100644 index 00000000000..3eca1ded4c7 --- /dev/null +++ b/.github/scripts/report-workflow-failure.cjs @@ -0,0 +1,86 @@ +const PATCH_WORKFLOWS = new Map([ + ['Update Runner Patches (Commit-to-Commit)', 'runner-patch-main'], + ['Update Runner Patches (Release-to-Release)', 'runner-patch-release'], +]); +const TRACKER_MARKER = ''; +const TRACKER_TITLE = 'Track runner patch-update failures'; + +/** Create or update the issue tracking a trusted failed workflow run. */ +async function reportWorkflowFailure({ github, context }) { + const run = context.payload.workflow_run; + const repository = `${context.repo.owner}/${context.repo.repo}`; + if (run.conclusion !== 'failure' || run.event === 'pull_request' || + run.head_repository?.full_name !== repository || + (!PATCH_WORKFLOWS.has(run.name) && !['Runner CI', 'Runner CD'].includes(run.name))) { + return; + } + + // REST listing avoids the search index lag between creating a tracker and the next report. + const issues = (await github.paginate(github.rest.issues.listForRepo, { + ...context.repo, state: 'all', per_page: 100, + })).filter(issue => !issue.pull_request).sort((a, b) => a.number - b.number); + + if (!PATCH_WORKFLOWS.has(run.name)) { + const marker = `workflow-run-id:${run.id}`; + if (issues.some(issue => issue.state === 'open' && + [...(issue.body || '').matchAll(/workflow-run-id:(\d+)/g)] + .some(match => Number(match[1]) === run.id))) { + return; + } + await createIssue(github, context.repo, `CI/CD Pipeline Failed: ${run.name}`, + `The GitHub Actions workflow **${run.name}** has failed.\n\n` + + `**Run URL:** ${run.html_url}\n**Run ID:** ${run.id}\n\n${marker}\n\n` + + 'Please investigate and resolve the issue.'); + return; + } + + const tracker = issues.find(issue => (issue.body || '').includes(TRACKER_MARKER)) || + issues.find(issue => issue.state === 'open' && + [...PATCH_WORKFLOWS.keys()].some(name => issue.title === `CI/CD Pipeline Failed: ${name}`)); + const key = PATCH_WORKFLOWS.get(run.name); + const start = ``; + const end = ``; + const section = new RegExp(`${start}[\\s\\S]*?${end}`); + let body = tracker?.body || ''; + const previous = body.match(section)?.[0]; + const previousRun = previous?.match(/workflow-run-id:(\d+) attempt:(\d+)/); + const attempt = run.run_attempt || 1; + if (previousRun && (Number(previousRun[1]) > run.id || + (Number(previousRun[1]) === run.id && Number(previousRun[2]) >= attempt))) { + return; + } + + const entry = `${start}\n### ${run.name}\n\n` + + `Latest reported failure: [run ${run.id}, attempt ${attempt}](${run.html_url})\n\n` + + `\n${end}`; + if (!body.includes(TRACKER_MARKER)) { + body += `${body ? '\n\n' : ''}${TRACKER_MARKER}\n\n` + + 'This issue tracks failures from both runner patch-update workflows. ' + + 'Close manually after confirming recovery; a successful scheduled run may be a no-op.'; + } + body = previous ? body.replace(section, () => entry) : `${body}\n\n${entry}`; + + if (tracker) { + await github.rest.issues.update({ + ...context.repo, issue_number: tracker.number, + title: TRACKER_TITLE, body, state: 'open', + }); + } else { + await createIssue(github, context.repo, TRACKER_TITLE, body); + } +} + +/** Create a labelled failure issue, provisioning the automation label if needed. */ +async function createIssue(github, repo, title, body) { + try { + await github.rest.issues.getLabel({ ...repo, name: 'ci-failure' }); + } catch (error) { + if (error.status !== 404) throw error; + await github.rest.issues.createLabel({ + ...repo, name: 'ci-failure', description: 'Automated CI/CD pipeline failure', color: 'B60205', + }); + } + await github.rest.issues.create({ ...repo, title, body, labels: ['bug', 'ci-failure'] }); +} + +module.exports = { reportWorkflowFailure }; diff --git a/.github/scripts/report-workflow-failure.test.cjs b/.github/scripts/report-workflow-failure.test.cjs new file mode 100644 index 00000000000..5f2bdc6e9a1 --- /dev/null +++ b/.github/scripts/report-workflow-failure.test.cjs @@ -0,0 +1,221 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { reportWorkflowFailure } = require('./report-workflow-failure.cjs'); + +const MAIN = 'Update Runner Patches (Commit-to-Commit)'; +const RELEASE = 'Update Runner Patches (Release-to-Release)'; +const TRACKER = ''; + +// Catch per-run deduplication accidentally being retained for patch failures. +test('shares one tracker across recurring failures of both patch workflows', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + await report(api, RELEASE, 501); + await report(api, MAIN, 502); + + assert.equal(api.issues.length, 1); + assert.equal(api.issues[0].title, 'Track runner patch-update failures'); + assert.deepEqual(api.issues[0].labels, ['bug', 'ci-failure']); + assert.match(api.issues[0].body, /runner-patch-main/); + assert.match(api.issues[0].body, /runner-patch-release/); + assert.match(api.issues[0].body, /actions\/runs\/501/); + assert.match(api.issues[0].body, /actions\/runs\/502/); + assert.doesNotMatch(api.issues[0].body, /actions\/runs\/500/); +}); + +// Keep migration attached to the oldest active issue and its existing Jira ticket. +test('adopts the oldest open exact-title legacy patch issue without closing others', async () => { + const api = fakeGithub([ + issue(249, `CI/CD Pipeline Failed: ${RELEASE}`), + issue(247, `CI/CD Pipeline Failed: ${MAIN}`, { state: 'closed' }), + issue(248, `CI/CD Pipeline Failed: ${MAIN}`, { body: 'Investigation and Jira link' }), + issue(246, `CI/CD Pipeline Failed: ${MAIN} extra`), + ]); + await report(api, RELEASE, 500); + + assert.equal(api.issues.length, 4); + assert.match(api.issues.find(i => i.number === 248).body, /Investigation and Jira link/); + assert.match(api.issues.find(i => i.number === 248).body, /runner-patch-failures/); + assert.equal(api.issues.find(i => i.number === 249).state, 'open'); + assert.equal(api.issues.find(i => i.number === 246).title, `CI/CD Pipeline Failed: ${MAIN} extra`); +}); + +// Prevent a closed tracker being replaced by a new issue or an unmarked legacy issue. +test('reopens the marked tracker on a new failure even when legacy issues remain', async () => { + const api = fakeGithub([issue(248, 'Tracker', { body: TRACKER, state: 'closed' }), + issue(247, `CI/CD Pipeline Failed: ${MAIN}`)]); + await report(api, MAIN, 500); + + assert.equal(api.issues.length, 2); + assert.equal(api.issues[0].state, 'open'); + assert.match(api.issues[0].body, /actions\/runs\/500/); + assert.equal(api.issues[1].title, `CI/CD Pipeline Failed: ${MAIN}`); +}); + +// Make retried deliveries idempotent and avoid reopening manually closed incidents. +test('does not write or reopen for identical or stale reports', async () => { + const api = fakeGithub(); + await report(api, MAIN, 502); + api.issues[0].state = 'closed'; + const writes = api.writes.length; + await report(api, MAIN, 502); + await report(api, MAIN, 501); + + assert.equal(api.writes.length, writes); + assert.equal(api.issues[0].state, 'closed'); + assert.match(api.issues[0].body, /actions\/runs\/502/); +}); + +// A rerun of an existing Actions run is a new failure even though the run ID is unchanged. +test('records a later run attempt and ignores a delayed earlier attempt', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + await report(api, MAIN, 500, { run_attempt: 2 }); + const writes = api.writes.length; + await report(api, MAIN, 500, { run_attempt: 1 }); + + assert.match(api.issues[0].body, /attempt:2/); + assert.equal(api.writes.length, writes); +}); + +// Updating one stream must not erase the other stream or human-authored investigation. +test('preserves human text outside managed entries and the other workflow entry', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + await report(api, RELEASE, 501); + api.issues[0].body = `Human introduction\n${api.issues[0].body}\nHuman follow-up`; + await report(api, MAIN, 502); + + assert.ok(api.issues[0].body.startsWith('Human introduction\n')); + assert.ok(api.issues[0].body.endsWith('\nHuman follow-up')); + assert.match(api.issues[0].body, /actions\/runs\/501/); + assert.match(api.issues[0].body, /actions\/runs\/502/); +}); + +// REST listing includes pull requests and can span multiple pages; neither may hide the tracker. +test('finds a tracker beyond the first page and excludes pull requests', async () => { + const existing = Array.from({ length: 100 }, (_, i) => issue(i + 1, 'Unrelated')); + existing[0].body = TRACKER; + existing[0].pull_request = { url: 'https://api.github.com/repos/canonical/github-actions-runner/pulls/1' }; + existing.push(issue(101, 'Tracker', { body: TRACKER })); + const api = fakeGithub(existing); + await report(api, MAIN, 500); + + assert.equal(api.issues.length, 101); + assert.equal(api.issues[0].title, 'Unrelated'); + assert.match(api.issues[100].body, /actions\/runs\/500/); + assert.equal(api.pages, 2); +}); + +// Ordinary build and release failures must retain their current per-run reporting behavior. +test('keeps Runner CI and Runner CD reporting separate and deduplicates retries', async () => { + for (const name of ['Runner CI', 'Runner CD']) { + const api = fakeGithub(); + await report(api, name, 500); + await report(api, name, 500); + await report(api, name, 501); + + assert.equal(api.issues.length, 2); + assert.equal(api.issues[0].title, `CI/CD Pipeline Failed: ${name}`); + assert.match(api.issues[0].body, /workflow-run-id:500/); + assert.doesNotMatch(api.issues[0].body, /runner-patch-failures/); + } +}); + +// Older reporter versions emitted literal backslash-n sequences rather than real line breaks. +test('recognizes legacy CI run markers without matching a longer run ID', async () => { + const api = fakeGithub([ + issue(248, 'CI/CD Pipeline Failed: Runner CI', { body: 'Failure\\n\\nworkflow-run-id:500\\nPlease investigate' }), + issue(249, 'CI/CD Pipeline Failed: Runner CI', { body: 'workflow-run-id:5010' }), + ]); + await report(api, 'Runner CI', 500); + assert.equal(api.issues.length, 2); + await report(api, 'Runner CI', 501); + assert.equal(api.issues.length, 3); + assert.match(api.issues[2].body, /workflow-run-id:501\n/); +}); + +// Untrusted or irrelevant workflow completions must not mutate repository issues. +test('ignores events outside the failure reporting boundary', async t => { + const cases = [ + ['successful run', { conclusion: 'success' }], + ['pull request', { event: 'pull_request' }], + ['fork run', { head_repository: { full_name: 'someone/runner' } }], + ['unknown workflow', { name: 'Other workflow' }], + ]; + for (const [name, overrides] of cases) { + await t.test(name, async () => { + const api = fakeGithub(); + await report(api, MAIN, 500, overrides); + assert.deepEqual(api.writes, []); + assert.equal(api.pages, 0); + }); + } +}); + +// API failures must surface to Actions instead of causing silent loss or duplicate creation. +test('propagates GitHub API errors', async t => { + for (const operation of ['listForRepo', 'getLabel', 'createLabel', 'create', 'update']) { + await t.test(operation, async () => { + const api = fakeGithub(operation === 'update' ? [issue(248, 'Tracker', { body: TRACKER })] : []); + api.github.rest.issues[operation] = async () => { throw new Error('GitHub unavailable'); }; + await assert.rejects(report(api, MAIN, 500), /GitHub unavailable/); + assert.equal(api.issues.length, operation === 'update' ? 1 : 0); + }); + } +}); + +function issue(number, title, overrides = {}) { + return { number, title, body: '', state: 'open', labels: ['bug'], ...overrides }; +} + +function report(api, name, id, overrides = {}) { + return reportWorkflowFailure({ github: api.github, context: { + repo: { owner: 'canonical', repo: 'github-actions-runner' }, + payload: { workflow_run: { + name, id, run_attempt: 1, html_url: `https://github.com/canonical/github-actions-runner/actions/runs/${id}`, + conclusion: 'failure', event: 'schedule', + head_repository: { full_name: 'canonical/github-actions-runner' }, ...overrides, + } }, + } }); +} + +function fakeGithub(existing = []) { + const api = { issues: structuredClone(existing), writes: [], pages: 0 }; + const labels = new Set(['bug']); + const issues = { + async listForRepo({ owner, repo, state, per_page, page = 1 }) { + assert.equal(`${owner}/${repo}`, 'canonical/github-actions-runner'); + assert.equal(state, 'all'); + api.pages++; + return { data: structuredClone(api.issues.slice((page - 1) * per_page, page * per_page)) }; + }, + async getLabel({ name }) { + if (!labels.has(name)) throw Object.assign(new Error('Not found'), { status: 404 }); + return { data: { name } }; + }, + async createLabel({ name }) { labels.add(name); return { data: { name } }; }, + async create(fields) { + const created = issue(Math.max(0, ...api.issues.map(i => i.number)) + 1, fields.title, fields); + api.issues.push(created); + api.writes.push(fields); + return { data: structuredClone(created) }; + }, + async update({ issue_number, ...fields }) { + const updated = api.issues.find(i => i.number === issue_number); + assert.ok(updated); + Object.assign(updated, fields); + api.writes.push(fields); + return { data: structuredClone(updated) }; + }, + }; + api.github = { rest: { issues }, async paginate(method, params) { + const result = []; + for (let page = 1; ; page++) { + const { data } = await method({ ...params, page }); + result.push(...data); + if (data.length < params.per_page) return result; + } + } }; + return api; +} diff --git a/.github/workflows/create-issue-on-failure.yaml b/.github/workflows/create-issue-on-failure.yaml index a29dbe0e51a..cd204a48cb6 100644 --- a/.github/workflows/create-issue-on-failure.yaml +++ b/.github/workflows/create-issue-on-failure.yaml @@ -16,6 +16,12 @@ permissions: issues: write contents: read +# Serialize shared issue writes without replacing pending failure reports. +concurrency: + group: workflow-failure-reporting + cancel-in-progress: false + queue: max + jobs: create-issue: # Only run this job if the triggered workflow concluded with a failure @@ -24,34 +30,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + persist-credentials: false - - name: Create GitHub Issue on Failure - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} - RUN_URL: ${{ github.event.workflow_run.html_url }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - run: | - ISSUE_TITLE="CI/CD Pipeline Failed: ${WORKFLOW_NAME}" - ISSUE_MARKER="workflow-run-id:${RUN_ID}" - ISSUE_BODY=$'The GitHub Actions workflow **'"${WORKFLOW_NAME}"'** has failed.\n\n**Run URL:** '"${RUN_URL}"'\n**Run ID:** '"${RUN_ID}"'\n\n'"${ISSUE_MARKER}"$'\n\nPlease investigate and resolve the issue.' - - # Check if an issue for this specific workflow run already exists to prevent duplicate spam on retries - EXISTING_ISSUE=$(gh issue list --repo "${REPOSITORY}" --state open --search "\"${ISSUE_MARKER}\" in:body" --json number --jq '.[0].number // empty') - if [ -n "$EXISTING_ISSUE" ]; then - echo "An issue for this workflow run already exists (#${EXISTING_ISSUE}). Skipping creation." - else - # gh issue create fails outright if a label is missing, so (re)create it first - gh label create "ci-failure" \ - --repo "${REPOSITORY}" \ - --description "Automated CI/CD pipeline failure" \ - --color "B60205" \ - --force - gh issue create \ - --repo "${REPOSITORY}" \ - --title "${ISSUE_TITLE}" \ - --body "${ISSUE_BODY}" \ - --label "bug" \ - --label "ci-failure" - fi + - name: Report workflow failure + uses: actions/github-script@v8 + with: + script: | + const { reportWorkflowFailure } = require('./.github/scripts/report-workflow-failure.cjs'); + await reportWorkflowFailure({ github, context }); diff --git a/.github/workflows/test-failure-reporting.yaml b/.github/workflows/test-failure-reporting.yaml new file mode 100644 index 00000000000..4922741ed10 --- /dev/null +++ b/.github/workflows/test-failure-reporting.yaml @@ -0,0 +1,29 @@ +name: Test workflow failure reporting + +on: + pull_request: + paths: + - .github/scripts/report-workflow-failure* + - .github/workflows/create-issue-on-failure.yaml + - .github/workflows/test-failure-reporting.yaml + push: + branches: [main] + paths: + - .github/scripts/report-workflow-failure* + - .github/workflows/create-issue-on-failure.yaml + - .github/workflows/test-failure-reporting.yaml + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: '24' + - run: node --test .github/scripts/report-workflow-failure.test.cjs diff --git a/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md b/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md new file mode 100644 index 00000000000..7ecdbe20b43 --- /dev/null +++ b/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md @@ -0,0 +1,58 @@ +# Runner Patch Tracker Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Reuse one GitHub/Jira tracking pair for patch-update failures rather than one pair per failed run. + +**Architecture:** Extract reporting into a dependency-free CommonJS module called from actions/github-script. Use paginated issue listing and stable body markers to identify the shared tracker, preserving human text outside a managed section. Serialize reporter runs and retain per-run reporting for unrelated CI/CD failures. + +**Tech Stack:** GitHub Actions, GitHub REST API via actions/github-script, Node.js built-in test runner. + +**Spec:** `docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md` + +## Global Constraints + +- Stable workflow identifiers: `runner-patch-main`, `runner-patch-release`. +- One shared patch tracker; reopen on recurrence; manual closure only. +- Adopt the oldest exact-title open legacy patch issue when no marked tracker exists. +- Preserve human text, other workflow entries, and existing Runner CI/CD behavior. +- Preserve same-repository/non-PR failure guards; add no runtime package dependencies. +- Do not close or modify backlog issues or Jira tickets during implementation. + +## Review Focus + +- A closed marked tracker must win over unmarked legacy issues. +- Issue listing must paginate and exclude pull requests. +- API failures must fail reporting rather than silently create duplicates. +- A retry or older run must not erase a newer failure or human investigation notes. +- Concurrent reporting must not cancel pending events or allow duplicate tracker creation. + +--- + +### Task 1: Implement and wire stable patch failure reporting + +**Files:** +- Create: `.github/scripts/report-workflow-failure.cjs` +- Create: `.github/scripts/report-workflow-failure.test.cjs` +- Modify: `.github/workflows/create-issue-on-failure.yaml` +- Create: `.github/workflows/test-failure-reporting.yaml` + +**Interfaces:** +- Consumes: `reportWorkflowFailure({ github, context })`, where `github` is the actions/github-script REST client and `context.payload.workflow_run` is the completed run. +- Produces: GitHub issue creation/update/reopen API calls, with stable tracker and managed-section body markers; no comments. + +- [ ] **Step 1: Write failing tests.** Use `node:test` and `node:assert/strict` with an in-memory GitHub API double. Assert one issue across repeated runs and both patch workflows; adoption of the oldest exact-title open legacy issue; reuse/reopen of a closed marked tracker; no write on an identical retry; preservation of human text and the second workflow's entry; no regression on an older run; paginated issue identity excluding PRs; failure propagation; unchanged Runner CI/CD per-run deduplication. +- [ ] **Step 2: Run red tests.** Run `node --test .github/scripts/report-workflow-failure.test.cjs`; expect failure because the module does not yet exist. +- [ ] **Step 3: Implement reporting.** Export `async function reportWorkflowFailure({ github, context })` from the CommonJS module. Use a stable shared tracker marker and managed per-workflow sections, paginated REST issue listing, exact legacy titles for adoption, current `bug`/`ci-failure` labels, and one issue update rather than comments. Retain the existing run-ID marker for ordinary CI/CD failures. Keep human text outside managed sections and compare run identities to ignore older events. +- [ ] **Step 4: Wire and test workflows.** Replace the inline reporter shell with actions/github-script loading the module. Preserve failure/source guards and permissions. Configure serialized reporting with a supported queue setting that does not drop pending runs; confirm current GitHub Actions syntax before selecting it. Add a path-filtered, read-only test workflow running the Node tests and automated assertions for the reporter guards and concurrency wiring. +- [ ] **Step 5: Run verification.** Run Node tests, actionlint for both touched workflows, and `git diff --check`; require all checks to pass. Review the diff for unexpected files and ensure no live issue mutations were used in tests. +- [ ] **Step 6: Review and raise PR.** Follow the repo PR template if present; perform an independent code review, address findings, commit as `fix(ci): group runner patch failures in one tracker`, push `fix/group-runner-patch-failures`, and open a PR. Explain adoption and that backlog cleanup is separate. + +## Execution status + +- [x] Approved design recorded. +- [x] Plan self-reviewed against the design. +- [x] Implementation approach confirmed (native execution). +- [ ] Task 1 complete. + +Steps 1–5 verified: 20 Node tests pass; source guards, permissions, module wiring, and max queue checked on parsed YAML; actionlint passes with only its unsupported `queue` schema diagnostic excluded; `git diff --check` passes. Independent review and PR publication remain. diff --git a/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md b/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md new file mode 100644 index 00000000000..67d82c6d9e9 --- /dev/null +++ b/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md @@ -0,0 +1,29 @@ +# Runner patch failure tracker + +## Intent + +Prevent repeated patch-update failures from creating a new GitHub issue and Jira ticket every day. The user approved one shared tracker for the two patch workflows, with existing duplicate cleanup reviewed separately after the reporting change merges. + +## Approaches + +- **Selected: one shared GitHub tracker.** Stable workflow identifiers distinguish the two patch streams inside one issue; the existing Jira sync continues to track that issue. +- One tracker per workflow improves separation but still produces two tickets. +- Jira-only grouping leaves the upstream GitHub issue spam and requires broader sync changes. + +## Behavior + +- Use `runner-patch-main` and `runner-patch-release` for the two patch workflows and a stable shared tracker marker. +- Find the marked issue across open and closed issues. When migrating, adopt the oldest open issue with an exact legacy patch-failure title, preserving its existing body and Jira link. The current candidate is canonical/github-actions-runner#248 (ISD-7256). +- Create a tracker only when none exists; reopen it on recurrence. +- Maintain separate latest-failure entries without daily comments. Preserve human-authored text outside the managed section and the other workflow's entry. +- Serialize tracker writes, including creation, without cancelling reporting events. Avoid relying on eventually consistent issue search for identity. +- Keep closure manual: a successful scheduled run can be a no-op, not evidence that patch processing recovered. +- Preserve existing run-based reporting for Runner CI and Runner CD. + +## Constraints and non-goals + +Use the existing GitHub Actions environment and API; do not modify runner runtime code or the Jira bot. Keep same-repository/non-PR failure guards and least-privilege permissions. No backlog closures, comments, or Jira changes in this PR. No new runtime package dependencies. + +## Verification + +Mock GitHub API calls to prove creation, adoption, repeated updates, both workflow entries, reopening, idempotent retries, pagination, preservation of human text, unrelated CI/CD behavior, and error propagation. Validate the workflow wiring and serialized reporting configuration. From df6c98ea01ae2781342a810c3c9465885d211cd8 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 10 Nov 2026 18:04:31 +0000 Subject: [PATCH 2/6] fix(ci): exempt patch tracker from stale closure --- .github/scripts/report-workflow-failure.cjs | 24 ++++++++++--- .../scripts/report-workflow-failure.test.cjs | 34 ++++++++++++++++--- .../plans/2026-11-10-runner-patch-tracker.md | 8 +++-- .../2026-11-10-runner-patch-tracker-design.md | 1 + 4 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.github/scripts/report-workflow-failure.cjs b/.github/scripts/report-workflow-failure.cjs index 3eca1ded4c7..d1158ef823e 100644 --- a/.github/scripts/report-workflow-failure.cjs +++ b/.github/scripts/report-workflow-failure.cjs @@ -61,26 +61,40 @@ async function reportWorkflowFailure({ github, context }) { body = previous ? body.replace(section, () => entry) : `${body}\n\n${entry}`; if (tracker) { + // The repository's stale bot must not resolve this long-lived tracker automatically. + if (!tracker.labels.some(label => (typeof label === 'string' ? label : label.name) === 'keep')) { + await ensureLabel(github, context.repo, 'keep'); + await github.rest.issues.addLabels({ ...context.repo, issue_number: tracker.number, labels: ['keep'] }); + } await github.rest.issues.update({ ...context.repo, issue_number: tracker.number, title: TRACKER_TITLE, body, state: 'open', }); } else { - await createIssue(github, context.repo, TRACKER_TITLE, body); + await createIssue(github, context.repo, TRACKER_TITLE, body, ['bug', 'ci-failure', 'keep']); } } /** Create a labelled failure issue, provisioning the automation label if needed. */ -async function createIssue(github, repo, title, body) { +async function createIssue(github, repo, title, body, labels = ['bug', 'ci-failure']) { + for (const name of labels.filter(name => name !== 'bug')) { + await ensureLabel(github, repo, name); + } + await github.rest.issues.create({ ...repo, title, body, labels }); +} + +/** Ensure an automation label exists without overwriting its repository configuration. */ +async function ensureLabel(github, repo, name) { try { - await github.rest.issues.getLabel({ ...repo, name: 'ci-failure' }); + await github.rest.issues.getLabel({ ...repo, name }); } catch (error) { if (error.status !== 404) throw error; await github.rest.issues.createLabel({ - ...repo, name: 'ci-failure', description: 'Automated CI/CD pipeline failure', color: 'B60205', + ...repo, name, + description: name === 'keep' ? 'Exempt from automatic stale closure' : 'Automated CI/CD pipeline failure', + color: name === 'keep' ? '0E8A16' : 'B60205', }); } - await github.rest.issues.create({ ...repo, title, body, labels: ['bug', 'ci-failure'] }); } module.exports = { reportWorkflowFailure }; diff --git a/.github/scripts/report-workflow-failure.test.cjs b/.github/scripts/report-workflow-failure.test.cjs index 5f2bdc6e9a1..a6d3fa01e93 100644 --- a/.github/scripts/report-workflow-failure.test.cjs +++ b/.github/scripts/report-workflow-failure.test.cjs @@ -15,7 +15,7 @@ test('shares one tracker across recurring failures of both patch workflows', asy assert.equal(api.issues.length, 1); assert.equal(api.issues[0].title, 'Track runner patch-update failures'); - assert.deepEqual(api.issues[0].labels, ['bug', 'ci-failure']); + assert.deepEqual(api.issues[0].labels, ['bug', 'ci-failure', 'keep']); assert.match(api.issues[0].body, /runner-patch-main/); assert.match(api.issues[0].body, /runner-patch-release/); assert.match(api.issues[0].body, /actions\/runs\/501/); @@ -40,6 +40,23 @@ test('adopts the oldest open exact-title legacy patch issue without closing othe assert.equal(api.issues.find(i => i.number === 246).title, `CI/CD Pipeline Failed: ${MAIN} extra`); }); +// Long-lived trackers must use the repository's stale-bot exemption without losing human labels. +test('exempts adopted and marked trackers from stale closure while preserving labels', async t => { + for (const [name, existing] of [ + ['legacy', issue(248, `CI/CD Pipeline Failed: ${MAIN}`, { labels: [{ name: 'bug' }, { name: 'investigating' }] })], + ['marked', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }] })], + ['already exempt', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'keep' }] })], + ]) { + await t.test(name, async () => { + const api = fakeGithub([existing]); + await report(api, MAIN, 500); + const names = api.issues[0].labels.map(label => typeof label === 'string' ? label : label.name); + assert.ok(names.includes('keep')); + for (const label of existing.labels) assert.ok(names.includes(label.name)); + }); + } +}); + // Prevent a closed tracker being replaced by a new issue or an unmarked legacy issue. test('reopens the marked tracker on a new failure even when legacy issues remain', async () => { const api = fakeGithub([issue(248, 'Tracker', { body: TRACKER, state: 'closed' }), @@ -117,6 +134,7 @@ test('keeps Runner CI and Runner CD reporting separate and deduplicates retries' assert.equal(api.issues.length, 2); assert.equal(api.issues[0].title, `CI/CD Pipeline Failed: ${name}`); + assert.deepEqual(api.issues[0].labels, ['bug', 'ci-failure']); assert.match(api.issues[0].body, /workflow-run-id:500/); assert.doesNotMatch(api.issues[0].body, /runner-patch-failures/); } @@ -155,12 +173,13 @@ test('ignores events outside the failure reporting boundary', async t => { // API failures must surface to Actions instead of causing silent loss or duplicate creation. test('propagates GitHub API errors', async t => { - for (const operation of ['listForRepo', 'getLabel', 'createLabel', 'create', 'update']) { + for (const operation of ['listForRepo', 'getLabel', 'createLabel', 'create', 'update', 'addLabels']) { await t.test(operation, async () => { - const api = fakeGithub(operation === 'update' ? [issue(248, 'Tracker', { body: TRACKER })] : []); + const existing = ['update', 'addLabels'].includes(operation); + const api = fakeGithub(existing ? [issue(248, 'Tracker', { body: TRACKER })] : []); api.github.rest.issues[operation] = async () => { throw new Error('GitHub unavailable'); }; await assert.rejects(report(api, MAIN, 500), /GitHub unavailable/); - assert.equal(api.issues.length, operation === 'update' ? 1 : 0); + assert.equal(api.issues.length, existing ? 1 : 0); }); } }); @@ -195,6 +214,13 @@ function fakeGithub(existing = []) { return { data: { name } }; }, async createLabel({ name }) { labels.add(name); return { data: { name } }; }, + async addLabels({ issue_number, labels: added }) { + const updated = api.issues.find(i => i.number === issue_number); + assert.ok(updated); + updated.labels.push(...added); + api.writes.push({ issue_number, labels: added }); + return { data: structuredClone(updated.labels) }; + }, async create(fields) { const created = issue(Math.max(0, ...api.issues.map(i => i.number)) + 1, fields.title, fields); api.issues.push(created); diff --git a/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md b/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md index 7ecdbe20b43..fa246e72f12 100644 --- a/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md +++ b/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md @@ -53,6 +53,10 @@ - [x] Approved design recorded. - [x] Plan self-reviewed against the design. - [x] Implementation approach confirmed (native execution). -- [ ] Task 1 complete. +- [x] Task 1 implementation and independent review complete. -Steps 1–5 verified: 20 Node tests pass; source guards, permissions, module wiring, and max queue checked on parsed YAML; actionlint passes with only its unsupported `queue` schema diagnostic excluded; `git diff --check` passes. Independent review and PR publication remain. +Steps 1–5 verified: 25 Node tests pass; source guards, permissions, module wiring, and max queue checked on parsed YAML; actionlint passes with only its unsupported `queue` schema diagnostic excluded; `git diff --check` passes. Independent review identified the repository's stale-bot exemption requirement; new regression tests failed before applying the existing `keep` label to patch trackers, then the suite passed. Publication is the final step. + +Full runtime suite: `cd src && ./dev.sh test` was attempted and stopped after approximately 12 minutes while provisioning external Node distributions, before any runtime tests ran. No .NET runtime code is changed. + +Review limitations: GitHub/Jira mutations and hosted queue execution were not tested live. The documented queue supports up to 100 pending reports. Preservation covers human text read before updating, not concurrent human edits during the API read/write window. diff --git a/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md b/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md index 67d82c6d9e9..5b8f7dca037 100644 --- a/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md +++ b/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md @@ -18,6 +18,7 @@ Prevent repeated patch-update failures from creating a new GitHub issue and Jira - Maintain separate latest-failure entries without daily comments. Preserve human-authored text outside the managed section and the other workflow's entry. - Serialize tracker writes, including creation, without cancelling reporting events. Avoid relying on eventually consistent issue search for identity. - Keep closure manual: a successful scheduled run can be a no-op, not evidence that patch processing recovered. +- Give patch trackers the existing `keep` label so the repository's stale bot does not close them automatically; preserve all other labels. - Preserve existing run-based reporting for Runner CI and Runner CD. ## Constraints and non-goals From 8f2c89691527b1eb3f128a72ddd737b1999260fe Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 10 Nov 2026 20:02:07 +0000 Subject: [PATCH 3/6] chore: remove internal process documents --- .../plans/2026-11-10-runner-patch-tracker.md | 62 ------------------- .../2026-11-10-runner-patch-tracker-design.md | 30 --------- 2 files changed, 92 deletions(-) delete mode 100644 docs/superpowers/plans/2026-11-10-runner-patch-tracker.md delete mode 100644 docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md diff --git a/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md b/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md deleted file mode 100644 index fa246e72f12..00000000000 --- a/docs/superpowers/plans/2026-11-10-runner-patch-tracker.md +++ /dev/null @@ -1,62 +0,0 @@ -# Runner Patch Tracker Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Reuse one GitHub/Jira tracking pair for patch-update failures rather than one pair per failed run. - -**Architecture:** Extract reporting into a dependency-free CommonJS module called from actions/github-script. Use paginated issue listing and stable body markers to identify the shared tracker, preserving human text outside a managed section. Serialize reporter runs and retain per-run reporting for unrelated CI/CD failures. - -**Tech Stack:** GitHub Actions, GitHub REST API via actions/github-script, Node.js built-in test runner. - -**Spec:** `docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md` - -## Global Constraints - -- Stable workflow identifiers: `runner-patch-main`, `runner-patch-release`. -- One shared patch tracker; reopen on recurrence; manual closure only. -- Adopt the oldest exact-title open legacy patch issue when no marked tracker exists. -- Preserve human text, other workflow entries, and existing Runner CI/CD behavior. -- Preserve same-repository/non-PR failure guards; add no runtime package dependencies. -- Do not close or modify backlog issues or Jira tickets during implementation. - -## Review Focus - -- A closed marked tracker must win over unmarked legacy issues. -- Issue listing must paginate and exclude pull requests. -- API failures must fail reporting rather than silently create duplicates. -- A retry or older run must not erase a newer failure or human investigation notes. -- Concurrent reporting must not cancel pending events or allow duplicate tracker creation. - ---- - -### Task 1: Implement and wire stable patch failure reporting - -**Files:** -- Create: `.github/scripts/report-workflow-failure.cjs` -- Create: `.github/scripts/report-workflow-failure.test.cjs` -- Modify: `.github/workflows/create-issue-on-failure.yaml` -- Create: `.github/workflows/test-failure-reporting.yaml` - -**Interfaces:** -- Consumes: `reportWorkflowFailure({ github, context })`, where `github` is the actions/github-script REST client and `context.payload.workflow_run` is the completed run. -- Produces: GitHub issue creation/update/reopen API calls, with stable tracker and managed-section body markers; no comments. - -- [ ] **Step 1: Write failing tests.** Use `node:test` and `node:assert/strict` with an in-memory GitHub API double. Assert one issue across repeated runs and both patch workflows; adoption of the oldest exact-title open legacy issue; reuse/reopen of a closed marked tracker; no write on an identical retry; preservation of human text and the second workflow's entry; no regression on an older run; paginated issue identity excluding PRs; failure propagation; unchanged Runner CI/CD per-run deduplication. -- [ ] **Step 2: Run red tests.** Run `node --test .github/scripts/report-workflow-failure.test.cjs`; expect failure because the module does not yet exist. -- [ ] **Step 3: Implement reporting.** Export `async function reportWorkflowFailure({ github, context })` from the CommonJS module. Use a stable shared tracker marker and managed per-workflow sections, paginated REST issue listing, exact legacy titles for adoption, current `bug`/`ci-failure` labels, and one issue update rather than comments. Retain the existing run-ID marker for ordinary CI/CD failures. Keep human text outside managed sections and compare run identities to ignore older events. -- [ ] **Step 4: Wire and test workflows.** Replace the inline reporter shell with actions/github-script loading the module. Preserve failure/source guards and permissions. Configure serialized reporting with a supported queue setting that does not drop pending runs; confirm current GitHub Actions syntax before selecting it. Add a path-filtered, read-only test workflow running the Node tests and automated assertions for the reporter guards and concurrency wiring. -- [ ] **Step 5: Run verification.** Run Node tests, actionlint for both touched workflows, and `git diff --check`; require all checks to pass. Review the diff for unexpected files and ensure no live issue mutations were used in tests. -- [ ] **Step 6: Review and raise PR.** Follow the repo PR template if present; perform an independent code review, address findings, commit as `fix(ci): group runner patch failures in one tracker`, push `fix/group-runner-patch-failures`, and open a PR. Explain adoption and that backlog cleanup is separate. - -## Execution status - -- [x] Approved design recorded. -- [x] Plan self-reviewed against the design. -- [x] Implementation approach confirmed (native execution). -- [x] Task 1 implementation and independent review complete. - -Steps 1–5 verified: 25 Node tests pass; source guards, permissions, module wiring, and max queue checked on parsed YAML; actionlint passes with only its unsupported `queue` schema diagnostic excluded; `git diff --check` passes. Independent review identified the repository's stale-bot exemption requirement; new regression tests failed before applying the existing `keep` label to patch trackers, then the suite passed. Publication is the final step. - -Full runtime suite: `cd src && ./dev.sh test` was attempted and stopped after approximately 12 minutes while provisioning external Node distributions, before any runtime tests ran. No .NET runtime code is changed. - -Review limitations: GitHub/Jira mutations and hosted queue execution were not tested live. The documented queue supports up to 100 pending reports. Preservation covers human text read before updating, not concurrent human edits during the API read/write window. diff --git a/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md b/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md deleted file mode 100644 index 5b8f7dca037..00000000000 --- a/docs/superpowers/specs/2026-11-10-runner-patch-tracker-design.md +++ /dev/null @@ -1,30 +0,0 @@ -# Runner patch failure tracker - -## Intent - -Prevent repeated patch-update failures from creating a new GitHub issue and Jira ticket every day. The user approved one shared tracker for the two patch workflows, with existing duplicate cleanup reviewed separately after the reporting change merges. - -## Approaches - -- **Selected: one shared GitHub tracker.** Stable workflow identifiers distinguish the two patch streams inside one issue; the existing Jira sync continues to track that issue. -- One tracker per workflow improves separation but still produces two tickets. -- Jira-only grouping leaves the upstream GitHub issue spam and requires broader sync changes. - -## Behavior - -- Use `runner-patch-main` and `runner-patch-release` for the two patch workflows and a stable shared tracker marker. -- Find the marked issue across open and closed issues. When migrating, adopt the oldest open issue with an exact legacy patch-failure title, preserving its existing body and Jira link. The current candidate is canonical/github-actions-runner#248 (ISD-7256). -- Create a tracker only when none exists; reopen it on recurrence. -- Maintain separate latest-failure entries without daily comments. Preserve human-authored text outside the managed section and the other workflow's entry. -- Serialize tracker writes, including creation, without cancelling reporting events. Avoid relying on eventually consistent issue search for identity. -- Keep closure manual: a successful scheduled run can be a no-op, not evidence that patch processing recovered. -- Give patch trackers the existing `keep` label so the repository's stale bot does not close them automatically; preserve all other labels. -- Preserve existing run-based reporting for Runner CI and Runner CD. - -## Constraints and non-goals - -Use the existing GitHub Actions environment and API; do not modify runner runtime code or the Jira bot. Keep same-repository/non-PR failure guards and least-privilege permissions. No backlog closures, comments, or Jira changes in this PR. No new runtime package dependencies. - -## Verification - -Mock GitHub API calls to prove creation, adoption, repeated updates, both workflow entries, reopening, idempotent retries, pagination, preservation of human text, unrelated CI/CD behavior, and error propagation. Validate the workflow wiring and serialized reporting configuration. From e81e3c1e6c0caa798f5c95bdc9077f40a760f2f8 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 10 Nov 2026 20:02:07 +0000 Subject: [PATCH 4/6] fix(ci): add missing labels when adopting patch trackers --- .github/scripts/report-workflow-failure.cjs | 13 +++++++++---- .github/scripts/report-workflow-failure.test.cjs | 8 +++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/scripts/report-workflow-failure.cjs b/.github/scripts/report-workflow-failure.cjs index d1158ef823e..04cf36b78d3 100644 --- a/.github/scripts/report-workflow-failure.cjs +++ b/.github/scripts/report-workflow-failure.cjs @@ -4,6 +4,7 @@ const PATCH_WORKFLOWS = new Map([ ]); const TRACKER_MARKER = ''; const TRACKER_TITLE = 'Track runner patch-update failures'; +const TRACKER_LABELS = ['bug', 'ci-failure', 'keep']; /** Create or update the issue tracking a trusted failed workflow run. */ async function reportWorkflowFailure({ github, context }) { @@ -62,16 +63,20 @@ async function reportWorkflowFailure({ github, context }) { if (tracker) { // The repository's stale bot must not resolve this long-lived tracker automatically. - if (!tracker.labels.some(label => (typeof label === 'string' ? label : label.name) === 'keep')) { - await ensureLabel(github, context.repo, 'keep'); - await github.rest.issues.addLabels({ ...context.repo, issue_number: tracker.number, labels: ['keep'] }); + const existingLabels = new Set(tracker.labels.map(label => typeof label === 'string' ? label : label.name)); + const missingLabels = TRACKER_LABELS.filter(name => !existingLabels.has(name)); + for (const name of missingLabels.filter(name => name !== 'bug')) { + await ensureLabel(github, context.repo, name); + } + if (missingLabels.length) { + await github.rest.issues.addLabels({ ...context.repo, issue_number: tracker.number, labels: missingLabels }); } await github.rest.issues.update({ ...context.repo, issue_number: tracker.number, title: TRACKER_TITLE, body, state: 'open', }); } else { - await createIssue(github, context.repo, TRACKER_TITLE, body, ['bug', 'ci-failure', 'keep']); + await createIssue(github, context.repo, TRACKER_TITLE, body, TRACKER_LABELS); } } diff --git a/.github/scripts/report-workflow-failure.test.cjs b/.github/scripts/report-workflow-failure.test.cjs index a6d3fa01e93..3bbb3778676 100644 --- a/.github/scripts/report-workflow-failure.test.cjs +++ b/.github/scripts/report-workflow-failure.test.cjs @@ -40,19 +40,21 @@ test('adopts the oldest open exact-title legacy patch issue without closing othe assert.equal(api.issues.find(i => i.number === 246).title, `CI/CD Pipeline Failed: ${MAIN} extra`); }); -// Long-lived trackers must use the repository's stale-bot exemption without losing human labels. -test('exempts adopted and marked trackers from stale closure while preserving labels', async t => { +// Migration must match new tracker labels without replacing human classification or stale exemptions. +test('adds missing standard labels to adopted and marked trackers while preserving labels', async t => { for (const [name, existing] of [ ['legacy', issue(248, `CI/CD Pipeline Failed: ${MAIN}`, { labels: [{ name: 'bug' }, { name: 'investigating' }] })], ['marked', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }] })], ['already exempt', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'keep' }] })], + ['fully labelled', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }, { name: 'ci-failure' }, { name: 'keep' }] })], ]) { await t.test(name, async () => { const api = fakeGithub([existing]); await report(api, MAIN, 500); const names = api.issues[0].labels.map(label => typeof label === 'string' ? label : label.name); - assert.ok(names.includes('keep')); + for (const required of ['bug', 'ci-failure', 'keep']) assert.ok(names.includes(required), required); for (const label of existing.labels) assert.ok(names.includes(label.name)); + if (name === 'fully labelled') assert.ok(api.writes.every(write => !write.labels)); }); } }); From f598cab2deaf5aca86365780daf7ea4654e292d6 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Tue, 10 Nov 2026 20:10:15 +0000 Subject: [PATCH 5/6] fix(ci): target ordinary failure deduplication searches --- .github/scripts/report-workflow-failure.cjs | 14 ++++--- .../scripts/report-workflow-failure.test.cjs | 38 ++++++++++++++++++- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/scripts/report-workflow-failure.cjs b/.github/scripts/report-workflow-failure.cjs index 04cf36b78d3..cfbda223eaf 100644 --- a/.github/scripts/report-workflow-failure.cjs +++ b/.github/scripts/report-workflow-failure.cjs @@ -16,14 +16,12 @@ async function reportWorkflowFailure({ github, context }) { return; } - // REST listing avoids the search index lag between creating a tracker and the next report. - const issues = (await github.paginate(github.rest.issues.listForRepo, { - ...context.repo, state: 'all', per_page: 100, - })).filter(issue => !issue.pull_request).sort((a, b) => a.number - b.number); - if (!PATCH_WORKFLOWS.has(run.name)) { const marker = `workflow-run-id:${run.id}`; - if (issues.some(issue => issue.state === 'open' && + const { data } = await github.rest.search.issuesAndPullRequests({ + q: `repo:${repository} is:issue is:open "${marker}" in:body`, per_page: 100, + }); + if (data.items.some(issue => [...(issue.body || '').matchAll(/workflow-run-id:(\d+)/g)] .some(match => Number(match[1]) === run.id))) { return; @@ -35,6 +33,10 @@ async function reportWorkflowFailure({ github, context }) { return; } + // REST listing avoids the search index lag between creating a tracker and the next report. + const issues = (await github.paginate(github.rest.issues.listForRepo, { + ...context.repo, state: 'all', per_page: 100, + })).filter(issue => !issue.pull_request).sort((a, b) => a.number - b.number); const tracker = issues.find(issue => (issue.body || '').includes(TRACKER_MARKER)) || issues.find(issue => issue.state === 'open' && [...PATCH_WORKFLOWS.keys()].some(name => issue.title === `CI/CD Pipeline Failed: ${name}`)); diff --git a/.github/scripts/report-workflow-failure.test.cjs b/.github/scripts/report-workflow-failure.test.cjs index 3bbb3778676..84eebd3fed4 100644 --- a/.github/scripts/report-workflow-failure.test.cjs +++ b/.github/scripts/report-workflow-failure.test.cjs @@ -142,6 +142,30 @@ test('keeps Runner CI and Runner CD reporting separate and deduplicates retries' } }); +// Ordinary failures must not pay for listing the repository's entire issue and pull request history. +test('uses a targeted run-marker lookup for ordinary failure deduplication', async () => { + for (const name of ['Runner CI', 'Runner CD']) { + const api = fakeGithub(); + api.github.rest.issues.listForRepo = async () => { throw new Error('Full issue listing is forbidden'); }; + await report(api, name, 500); + await report(api, name, 500); + + assert.equal(api.issues.length, 1); + assert.deepEqual(api.searches, [ + 'repo:canonical/github-actions-runner is:issue is:open "workflow-run-id:500" in:body', + 'repo:canonical/github-actions-runner is:issue is:open "workflow-run-id:500" in:body', + ]); + } +}); + +// Search failures must not be interpreted as an absent ordinary failure ticket. +test('propagates ordinary deduplication search errors without creating an issue', async () => { + const api = fakeGithub(); + api.github.rest.search.issuesAndPullRequests = async () => { throw new Error('Search unavailable'); }; + await assert.rejects(report(api, 'Runner CI', 500), /Search unavailable/); + assert.deepEqual(api.issues, []); +}); + // Older reporter versions emitted literal backslash-n sequences rather than real line breaks. test('recognizes legacy CI run markers without matching a longer run ID', async () => { const api = fakeGithub([ @@ -202,7 +226,7 @@ function report(api, name, id, overrides = {}) { } function fakeGithub(existing = []) { - const api = { issues: structuredClone(existing), writes: [], pages: 0 }; + const api = { issues: structuredClone(existing), writes: [], pages: 0, searches: [] }; const labels = new Set(['bug']); const issues = { async listForRepo({ owner, repo, state, per_page, page = 1 }) { @@ -237,7 +261,17 @@ function fakeGithub(existing = []) { return { data: structuredClone(updated) }; }, }; - api.github = { rest: { issues }, async paginate(method, params) { + const search = { + async issuesAndPullRequests({ q, per_page }) { + api.searches.push(q); + const marker = q.match(/"(workflow-run-id:\d+)"/)[1]; + const matches = api.issues.filter(issue => issue.state === 'open' && !issue.pull_request && + (issue.body || '').includes(marker)); + return { data: { total_count: matches.length, incomplete_results: false, + items: structuredClone(matches.slice(0, per_page)) } }; + }, + }; + api.github = { rest: { issues, search }, async paginate(method, params) { const result = []; for (let page = 1; ; page++) { const { data } = await method({ ...params, page }); From 3ac4032c6fe77f12b8b00c359daaef28bbd8bb62 Mon Sep 17 00:00:00 2001 From: Christopher Bartz Date: Fri, 13 Nov 2026 08:03:52 +0000 Subject: [PATCH 6/6] fix(ci): create a new patch tracker for each incident --- .github/scripts/report-workflow-failure.cjs | 28 +++-- .../scripts/report-workflow-failure.test.cjs | 100 +++++++++++++++--- 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/.github/scripts/report-workflow-failure.cjs b/.github/scripts/report-workflow-failure.cjs index cfbda223eaf..47f62069b5c 100644 --- a/.github/scripts/report-workflow-failure.cjs +++ b/.github/scripts/report-workflow-failure.cjs @@ -37,29 +37,35 @@ async function reportWorkflowFailure({ github, context }) { const issues = (await github.paginate(github.rest.issues.listForRepo, { ...context.repo, state: 'all', per_page: 100, })).filter(issue => !issue.pull_request).sort((a, b) => a.number - b.number); - const tracker = issues.find(issue => (issue.body || '').includes(TRACKER_MARKER)) || - issues.find(issue => issue.state === 'open' && - [...PATCH_WORKFLOWS.keys()].some(name => issue.title === `CI/CD Pipeline Failed: ${name}`)); + const markedTrackers = issues.filter(issue => (issue.body || '').includes(TRACKER_MARKER)); + const tracker = markedTrackers.find(issue => issue.state === 'open'); const key = PATCH_WORKFLOWS.get(run.name); const start = ``; const end = ``; const section = new RegExp(`${start}[\\s\\S]*?${end}`); let body = tracker?.body || ''; const previous = body.match(section)?.[0]; - const previousRun = previous?.match(/workflow-run-id:(\d+) attempt:(\d+)/); const attempt = run.run_attempt || 1; - if (previousRun && (Number(previousRun[1]) > run.id || - (Number(previousRun[1]) === run.id && Number(previousRun[2]) >= attempt))) { - return; + // Closed incidents still identify replays, but must never become update targets. + for (const incident of markedTrackers) { + const previousRun = incident.body.match(section)?.[0].match(/workflow-run-id:(\d+) attempt:(\d+)/); + if (previousRun && (Number(previousRun[1]) > run.id || + (Number(previousRun[1]) === run.id && Number(previousRun[2]) >= attempt))) { + return; + } } const entry = `${start}\n### ${run.name}\n\n` + `Latest reported failure: [run ${run.id}, attempt ${attempt}](${run.html_url})\n\n` + `\n${end}`; - if (!body.includes(TRACKER_MARKER)) { - body += `${body ? '\n\n' : ''}${TRACKER_MARKER}\n\n` + - 'This issue tracks failures from both runner patch-update workflows. ' + + if (!tracker) { + body = `${TRACKER_MARKER}\n\n` + + 'This issue tracks the current incident across both runner patch-update workflows. ' + 'Close manually after confirming recovery; a successful scheduled run may be a no-op.'; + const previousIncident = markedTrackers.findLast(issue => issue.state === 'closed'); + if (previousIncident) { + body += `\n\nPrevious incident: https://github.com/${repository}/issues/${previousIncident.number}`; + } } body = previous ? body.replace(section, () => entry) : `${body}\n\n${entry}`; @@ -75,7 +81,7 @@ async function reportWorkflowFailure({ github, context }) { } await github.rest.issues.update({ ...context.repo, issue_number: tracker.number, - title: TRACKER_TITLE, body, state: 'open', + title: TRACKER_TITLE, body, }); } else { await createIssue(github, context.repo, TRACKER_TITLE, body, TRACKER_LABELS); diff --git a/.github/scripts/report-workflow-failure.test.cjs b/.github/scripts/report-workflow-failure.test.cjs index 84eebd3fed4..3bdc8aea1ae 100644 --- a/.github/scripts/report-workflow-failure.test.cjs +++ b/.github/scripts/report-workflow-failure.test.cjs @@ -23,27 +23,27 @@ test('shares one tracker across recurring failures of both patch workflows', asy assert.doesNotMatch(api.issues[0].body, /actions\/runs\/500/); }); -// Keep migration attached to the oldest active issue and its existing Jira ticket. -test('adopts the oldest open exact-title legacy patch issue without closing others', async () => { +// Old daily failure issues are not incident trackers and must remain untouched. +test('creates a new marked tracker instead of adopting unmarked legacy issues', async () => { const api = fakeGithub([ issue(249, `CI/CD Pipeline Failed: ${RELEASE}`), issue(247, `CI/CD Pipeline Failed: ${MAIN}`, { state: 'closed' }), issue(248, `CI/CD Pipeline Failed: ${MAIN}`, { body: 'Investigation and Jira link' }), issue(246, `CI/CD Pipeline Failed: ${MAIN} extra`), ]); + const before = structuredClone(api.issues); await report(api, RELEASE, 500); - assert.equal(api.issues.length, 4); - assert.match(api.issues.find(i => i.number === 248).body, /Investigation and Jira link/); - assert.match(api.issues.find(i => i.number === 248).body, /runner-patch-failures/); - assert.equal(api.issues.find(i => i.number === 249).state, 'open'); - assert.equal(api.issues.find(i => i.number === 246).title, `CI/CD Pipeline Failed: ${MAIN} extra`); + assert.equal(api.issues.length, 5); + assert.deepEqual(api.issues.slice(0, 4), before); + assert.match(api.issues[4].body, /runner-patch-failures/); + assert.match(api.issues[4].body, /actions\/runs\/500/); }); -// Migration must match new tracker labels without replacing human classification or stale exemptions. -test('adds missing standard labels to adopted and marked trackers while preserving labels', async t => { +// Active trackers must retain human classification and stale exemptions while receiving standard labels. +test('adds missing standard labels to open marked trackers while preserving labels', async t => { for (const [name, existing] of [ - ['legacy', issue(248, `CI/CD Pipeline Failed: ${MAIN}`, { labels: [{ name: 'bug' }, { name: 'investigating' }] })], + ['human-labelled', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }, { name: 'investigating' }] })], ['marked', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }] })], ['already exempt', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'keep' }] })], ['fully labelled', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }, { name: 'ci-failure' }, { name: 'keep' }] })], @@ -59,16 +59,58 @@ test('adds missing standard labels to adopted and marked trackers while preservi } }); -// Prevent a closed tracker being replaced by a new issue or an unmarked legacy issue. -test('reopens the marked tracker on a new failure even when legacy issues remain', async () => { +// A closed incident must remain closed, including when old daily duplicates are still open. +test('creates a new tracker for a new incident without changing closed or legacy issues', async () => { const api = fakeGithub([issue(248, 'Tracker', { body: TRACKER, state: 'closed' }), issue(247, `CI/CD Pipeline Failed: ${MAIN}`)]); + const before = structuredClone(api.issues); + await report(api, MAIN, 500); + + assert.equal(api.issues.length, 3); + assert.deepEqual(api.issues.slice(0, 2), before); + assert.match(api.issues[2].body, /actions\/runs\/500/); + assert.match(api.issues[2].body, /Previous incident: https:\/\/github.com\/canonical\/github-actions-runner\/issues\/248/); +}); + +// An older closed marker must not hide an existing active tracker. +test('reuses an open marked tracker rather than an older closed marked tracker', async () => { + const api = fakeGithub([ + issue(248, 'Closed incident', { body: TRACKER, state: 'closed' }), + issue(249, 'Active incident', { body: TRACKER }), + ]); + const closed = structuredClone(api.issues[0]); await report(api, MAIN, 500); assert.equal(api.issues.length, 2); - assert.equal(api.issues[0].state, 'open'); + assert.deepEqual(api.issues[0], closed); + assert.match(api.issues[1].body, /actions\/runs\/500/); +}); + +// Later incidents link to their nearest predecessor, not the oldest historical tracker. +test('links a new tracker to the most recent closed marked incident', async () => { + const api = fakeGithub([ + issue(248, 'Older incident', { body: TRACKER, state: 'closed' }), + issue(249, 'Most recent incident', { body: TRACKER, state: 'closed' }), + ]); + const before = structuredClone(api.issues); + await report(api, MAIN, 500); + + assert.deepEqual(api.issues.slice(0, 2), before); + assert.match(api.issues[2].body, /Previous incident: https:\/\/github.com\/canonical\/github-actions-runner\/issues\/249/); +}); + +// The reporter must not undo a human close that races with its body update. +test('does not request reopening when updating an open tracker', async () => { + const api = fakeGithub([issue(248, 'Active incident', { body: TRACKER })]); + const update = api.github.rest.issues.update; + api.github.rest.issues.update = async fields => { + api.issues[0].state = 'closed'; + return update(fields); + }; + await report(api, MAIN, 500); + + assert.equal(api.issues[0].state, 'closed'); assert.match(api.issues[0].body, /actions\/runs\/500/); - assert.equal(api.issues[1].title, `CI/CD Pipeline Failed: ${MAIN}`); }); // Make retried deliveries idempotent and avoid reopening manually closed incidents. @@ -85,6 +127,36 @@ test('does not write or reopen for identical or stale reports', async () => { assert.match(api.issues[0].body, /actions\/runs\/502/); }); +// Replays from an earlier incident must not create another ticket or contaminate a new stream entry. +test('ignores events recorded in a closed incident while another incident is open', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + api.issues[0].state = 'closed'; + await report(api, RELEASE, 501); + const before = structuredClone(api.issues); + const writes = api.writes.length; + await report(api, MAIN, 500); + await report(api, MAIN, 499); + + assert.equal(api.issues.length, 2); + assert.deepEqual(api.issues, before); + assert.equal(api.writes.length, writes); + assert.doesNotMatch(api.issues[1].body, /runner-patch-main/); +}); + +// A new failed attempt after resolution is a fresh incident, even if its Actions run ID is unchanged. +test('creates a new tracker for a later attempt after the previous tracker is closed', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + api.issues[0].state = 'closed'; + const closed = structuredClone(api.issues[0]); + await report(api, MAIN, 500, { run_attempt: 2 }); + + assert.equal(api.issues.length, 2); + assert.deepEqual(api.issues[0], closed); + assert.match(api.issues[1].body, /attempt:2/); +}); + // A rerun of an existing Actions run is a new failure even though the run ID is unchanged. test('records a later run attempt and ignores a delayed earlier attempt', async () => { const api = fakeGithub();