diff --git a/.github/scripts/report-workflow-failure.cjs b/.github/scripts/report-workflow-failure.cjs new file mode 100644 index 00000000000..47f62069b5c --- /dev/null +++ b/.github/scripts/report-workflow-failure.cjs @@ -0,0 +1,113 @@ +const PATCH_WORKFLOWS = new Map([ + ['Update Runner Patches (Commit-to-Commit)', 'runner-patch-main'], + ['Update Runner Patches (Release-to-Release)', 'runner-patch-release'], +]); +const TRACKER_MARKER = ''; +const TRACKER_TITLE = 'Track runner patch-update failures'; +const TRACKER_LABELS = ['bug', 'ci-failure', 'keep']; + +/** Create or update the issue tracking a trusted failed workflow run. */ +async function reportWorkflowFailure({ github, context }) { + const run = context.payload.workflow_run; + const repository = `${context.repo.owner}/${context.repo.repo}`; + if (run.conclusion !== 'failure' || run.event === 'pull_request' || + run.head_repository?.full_name !== repository || + (!PATCH_WORKFLOWS.has(run.name) && !['Runner CI', 'Runner CD'].includes(run.name))) { + return; + } + + if (!PATCH_WORKFLOWS.has(run.name)) { + const marker = `workflow-run-id:${run.id}`; + const { data } = await github.rest.search.issuesAndPullRequests({ + q: `repo:${repository} is:issue is:open "${marker}" in:body`, per_page: 100, + }); + if (data.items.some(issue => + [...(issue.body || '').matchAll(/workflow-run-id:(\d+)/g)] + .some(match => Number(match[1]) === run.id))) { + return; + } + await createIssue(github, context.repo, `CI/CD Pipeline Failed: ${run.name}`, + `The GitHub Actions workflow **${run.name}** has failed.\n\n` + + `**Run URL:** ${run.html_url}\n**Run ID:** ${run.id}\n\n${marker}\n\n` + + 'Please investigate and resolve the issue.'); + return; + } + + // REST listing avoids the search index lag between creating a tracker and the next report. + const issues = (await github.paginate(github.rest.issues.listForRepo, { + ...context.repo, state: 'all', per_page: 100, + })).filter(issue => !issue.pull_request).sort((a, b) => a.number - b.number); + const markedTrackers = issues.filter(issue => (issue.body || '').includes(TRACKER_MARKER)); + const tracker = markedTrackers.find(issue => issue.state === 'open'); + const key = PATCH_WORKFLOWS.get(run.name); + const start = ``; + const end = ``; + const section = new RegExp(`${start}[\\s\\S]*?${end}`); + let body = tracker?.body || ''; + const previous = body.match(section)?.[0]; + const attempt = run.run_attempt || 1; + // Closed incidents still identify replays, but must never become update targets. + for (const incident of markedTrackers) { + const previousRun = incident.body.match(section)?.[0].match(/workflow-run-id:(\d+) attempt:(\d+)/); + if (previousRun && (Number(previousRun[1]) > run.id || + (Number(previousRun[1]) === run.id && Number(previousRun[2]) >= attempt))) { + return; + } + } + + const entry = `${start}\n### ${run.name}\n\n` + + `Latest reported failure: [run ${run.id}, attempt ${attempt}](${run.html_url})\n\n` + + `\n${end}`; + if (!tracker) { + body = `${TRACKER_MARKER}\n\n` + + 'This issue tracks the current incident across both runner patch-update workflows. ' + + 'Close manually after confirming recovery; a successful scheduled run may be a no-op.'; + const previousIncident = markedTrackers.findLast(issue => issue.state === 'closed'); + if (previousIncident) { + body += `\n\nPrevious incident: https://github.com/${repository}/issues/${previousIncident.number}`; + } + } + body = previous ? body.replace(section, () => entry) : `${body}\n\n${entry}`; + + if (tracker) { + // The repository's stale bot must not resolve this long-lived tracker automatically. + const existingLabels = new Set(tracker.labels.map(label => typeof label === 'string' ? label : label.name)); + const missingLabels = TRACKER_LABELS.filter(name => !existingLabels.has(name)); + for (const name of missingLabels.filter(name => name !== 'bug')) { + await ensureLabel(github, context.repo, name); + } + if (missingLabels.length) { + await github.rest.issues.addLabels({ ...context.repo, issue_number: tracker.number, labels: missingLabels }); + } + await github.rest.issues.update({ + ...context.repo, issue_number: tracker.number, + title: TRACKER_TITLE, body, + }); + } else { + await createIssue(github, context.repo, TRACKER_TITLE, body, TRACKER_LABELS); + } +} + +/** Create a labelled failure issue, provisioning the automation label if needed. */ +async function createIssue(github, repo, title, body, labels = ['bug', 'ci-failure']) { + for (const name of labels.filter(name => name !== 'bug')) { + await ensureLabel(github, repo, name); + } + await github.rest.issues.create({ ...repo, title, body, labels }); +} + +/** Ensure an automation label exists without overwriting its repository configuration. */ +async function ensureLabel(github, repo, name) { + try { + await github.rest.issues.getLabel({ ...repo, name }); + } catch (error) { + if (error.status !== 404) throw error; + await github.rest.issues.createLabel({ + ...repo, name, + description: name === 'keep' ? 'Exempt from automatic stale closure' : 'Automated CI/CD pipeline failure', + color: name === 'keep' ? '0E8A16' : 'B60205', + }); + } +} + +module.exports = { reportWorkflowFailure }; diff --git a/.github/scripts/report-workflow-failure.test.cjs b/.github/scripts/report-workflow-failure.test.cjs new file mode 100644 index 00000000000..3bdc8aea1ae --- /dev/null +++ b/.github/scripts/report-workflow-failure.test.cjs @@ -0,0 +1,355 @@ +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { reportWorkflowFailure } = require('./report-workflow-failure.cjs'); + +const MAIN = 'Update Runner Patches (Commit-to-Commit)'; +const RELEASE = 'Update Runner Patches (Release-to-Release)'; +const TRACKER = ''; + +// Catch per-run deduplication accidentally being retained for patch failures. +test('shares one tracker across recurring failures of both patch workflows', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + await report(api, RELEASE, 501); + await report(api, MAIN, 502); + + assert.equal(api.issues.length, 1); + assert.equal(api.issues[0].title, 'Track runner patch-update failures'); + assert.deepEqual(api.issues[0].labels, ['bug', 'ci-failure', 'keep']); + assert.match(api.issues[0].body, /runner-patch-main/); + assert.match(api.issues[0].body, /runner-patch-release/); + assert.match(api.issues[0].body, /actions\/runs\/501/); + assert.match(api.issues[0].body, /actions\/runs\/502/); + assert.doesNotMatch(api.issues[0].body, /actions\/runs\/500/); +}); + +// Old daily failure issues are not incident trackers and must remain untouched. +test('creates a new marked tracker instead of adopting unmarked legacy issues', async () => { + const api = fakeGithub([ + issue(249, `CI/CD Pipeline Failed: ${RELEASE}`), + issue(247, `CI/CD Pipeline Failed: ${MAIN}`, { state: 'closed' }), + issue(248, `CI/CD Pipeline Failed: ${MAIN}`, { body: 'Investigation and Jira link' }), + issue(246, `CI/CD Pipeline Failed: ${MAIN} extra`), + ]); + const before = structuredClone(api.issues); + await report(api, RELEASE, 500); + + assert.equal(api.issues.length, 5); + assert.deepEqual(api.issues.slice(0, 4), before); + assert.match(api.issues[4].body, /runner-patch-failures/); + assert.match(api.issues[4].body, /actions\/runs\/500/); +}); + +// Active trackers must retain human classification and stale exemptions while receiving standard labels. +test('adds missing standard labels to open marked trackers while preserving labels', async t => { + for (const [name, existing] of [ + ['human-labelled', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }, { name: 'investigating' }] })], + ['marked', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }] })], + ['already exempt', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'keep' }] })], + ['fully labelled', issue(248, 'Tracker', { body: TRACKER, labels: [{ name: 'bug' }, { name: 'ci-failure' }, { name: 'keep' }] })], + ]) { + await t.test(name, async () => { + const api = fakeGithub([existing]); + await report(api, MAIN, 500); + const names = api.issues[0].labels.map(label => typeof label === 'string' ? label : label.name); + for (const required of ['bug', 'ci-failure', 'keep']) assert.ok(names.includes(required), required); + for (const label of existing.labels) assert.ok(names.includes(label.name)); + if (name === 'fully labelled') assert.ok(api.writes.every(write => !write.labels)); + }); + } +}); + +// A closed incident must remain closed, including when old daily duplicates are still open. +test('creates a new tracker for a new incident without changing closed or legacy issues', async () => { + const api = fakeGithub([issue(248, 'Tracker', { body: TRACKER, state: 'closed' }), + issue(247, `CI/CD Pipeline Failed: ${MAIN}`)]); + const before = structuredClone(api.issues); + await report(api, MAIN, 500); + + assert.equal(api.issues.length, 3); + assert.deepEqual(api.issues.slice(0, 2), before); + assert.match(api.issues[2].body, /actions\/runs\/500/); + assert.match(api.issues[2].body, /Previous incident: https:\/\/github.com\/canonical\/github-actions-runner\/issues\/248/); +}); + +// An older closed marker must not hide an existing active tracker. +test('reuses an open marked tracker rather than an older closed marked tracker', async () => { + const api = fakeGithub([ + issue(248, 'Closed incident', { body: TRACKER, state: 'closed' }), + issue(249, 'Active incident', { body: TRACKER }), + ]); + const closed = structuredClone(api.issues[0]); + await report(api, MAIN, 500); + + assert.equal(api.issues.length, 2); + assert.deepEqual(api.issues[0], closed); + assert.match(api.issues[1].body, /actions\/runs\/500/); +}); + +// Later incidents link to their nearest predecessor, not the oldest historical tracker. +test('links a new tracker to the most recent closed marked incident', async () => { + const api = fakeGithub([ + issue(248, 'Older incident', { body: TRACKER, state: 'closed' }), + issue(249, 'Most recent incident', { body: TRACKER, state: 'closed' }), + ]); + const before = structuredClone(api.issues); + await report(api, MAIN, 500); + + assert.deepEqual(api.issues.slice(0, 2), before); + assert.match(api.issues[2].body, /Previous incident: https:\/\/github.com\/canonical\/github-actions-runner\/issues\/249/); +}); + +// The reporter must not undo a human close that races with its body update. +test('does not request reopening when updating an open tracker', async () => { + const api = fakeGithub([issue(248, 'Active incident', { body: TRACKER })]); + const update = api.github.rest.issues.update; + api.github.rest.issues.update = async fields => { + api.issues[0].state = 'closed'; + return update(fields); + }; + await report(api, MAIN, 500); + + assert.equal(api.issues[0].state, 'closed'); + assert.match(api.issues[0].body, /actions\/runs\/500/); +}); + +// Make retried deliveries idempotent and avoid reopening manually closed incidents. +test('does not write or reopen for identical or stale reports', async () => { + const api = fakeGithub(); + await report(api, MAIN, 502); + api.issues[0].state = 'closed'; + const writes = api.writes.length; + await report(api, MAIN, 502); + await report(api, MAIN, 501); + + assert.equal(api.writes.length, writes); + assert.equal(api.issues[0].state, 'closed'); + assert.match(api.issues[0].body, /actions\/runs\/502/); +}); + +// Replays from an earlier incident must not create another ticket or contaminate a new stream entry. +test('ignores events recorded in a closed incident while another incident is open', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + api.issues[0].state = 'closed'; + await report(api, RELEASE, 501); + const before = structuredClone(api.issues); + const writes = api.writes.length; + await report(api, MAIN, 500); + await report(api, MAIN, 499); + + assert.equal(api.issues.length, 2); + assert.deepEqual(api.issues, before); + assert.equal(api.writes.length, writes); + assert.doesNotMatch(api.issues[1].body, /runner-patch-main/); +}); + +// A new failed attempt after resolution is a fresh incident, even if its Actions run ID is unchanged. +test('creates a new tracker for a later attempt after the previous tracker is closed', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + api.issues[0].state = 'closed'; + const closed = structuredClone(api.issues[0]); + await report(api, MAIN, 500, { run_attempt: 2 }); + + assert.equal(api.issues.length, 2); + assert.deepEqual(api.issues[0], closed); + assert.match(api.issues[1].body, /attempt:2/); +}); + +// A rerun of an existing Actions run is a new failure even though the run ID is unchanged. +test('records a later run attempt and ignores a delayed earlier attempt', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + await report(api, MAIN, 500, { run_attempt: 2 }); + const writes = api.writes.length; + await report(api, MAIN, 500, { run_attempt: 1 }); + + assert.match(api.issues[0].body, /attempt:2/); + assert.equal(api.writes.length, writes); +}); + +// Updating one stream must not erase the other stream or human-authored investigation. +test('preserves human text outside managed entries and the other workflow entry', async () => { + const api = fakeGithub(); + await report(api, MAIN, 500); + await report(api, RELEASE, 501); + api.issues[0].body = `Human introduction\n${api.issues[0].body}\nHuman follow-up`; + await report(api, MAIN, 502); + + assert.ok(api.issues[0].body.startsWith('Human introduction\n')); + assert.ok(api.issues[0].body.endsWith('\nHuman follow-up')); + assert.match(api.issues[0].body, /actions\/runs\/501/); + assert.match(api.issues[0].body, /actions\/runs\/502/); +}); + +// REST listing includes pull requests and can span multiple pages; neither may hide the tracker. +test('finds a tracker beyond the first page and excludes pull requests', async () => { + const existing = Array.from({ length: 100 }, (_, i) => issue(i + 1, 'Unrelated')); + existing[0].body = TRACKER; + existing[0].pull_request = { url: 'https://api.github.com/repos/canonical/github-actions-runner/pulls/1' }; + existing.push(issue(101, 'Tracker', { body: TRACKER })); + const api = fakeGithub(existing); + await report(api, MAIN, 500); + + assert.equal(api.issues.length, 101); + assert.equal(api.issues[0].title, 'Unrelated'); + assert.match(api.issues[100].body, /actions\/runs\/500/); + assert.equal(api.pages, 2); +}); + +// Ordinary build and release failures must retain their current per-run reporting behavior. +test('keeps Runner CI and Runner CD reporting separate and deduplicates retries', async () => { + for (const name of ['Runner CI', 'Runner CD']) { + const api = fakeGithub(); + await report(api, name, 500); + await report(api, name, 500); + await report(api, name, 501); + + assert.equal(api.issues.length, 2); + assert.equal(api.issues[0].title, `CI/CD Pipeline Failed: ${name}`); + assert.deepEqual(api.issues[0].labels, ['bug', 'ci-failure']); + assert.match(api.issues[0].body, /workflow-run-id:500/); + assert.doesNotMatch(api.issues[0].body, /runner-patch-failures/); + } +}); + +// Ordinary failures must not pay for listing the repository's entire issue and pull request history. +test('uses a targeted run-marker lookup for ordinary failure deduplication', async () => { + for (const name of ['Runner CI', 'Runner CD']) { + const api = fakeGithub(); + api.github.rest.issues.listForRepo = async () => { throw new Error('Full issue listing is forbidden'); }; + await report(api, name, 500); + await report(api, name, 500); + + assert.equal(api.issues.length, 1); + assert.deepEqual(api.searches, [ + 'repo:canonical/github-actions-runner is:issue is:open "workflow-run-id:500" in:body', + 'repo:canonical/github-actions-runner is:issue is:open "workflow-run-id:500" in:body', + ]); + } +}); + +// Search failures must not be interpreted as an absent ordinary failure ticket. +test('propagates ordinary deduplication search errors without creating an issue', async () => { + const api = fakeGithub(); + api.github.rest.search.issuesAndPullRequests = async () => { throw new Error('Search unavailable'); }; + await assert.rejects(report(api, 'Runner CI', 500), /Search unavailable/); + assert.deepEqual(api.issues, []); +}); + +// Older reporter versions emitted literal backslash-n sequences rather than real line breaks. +test('recognizes legacy CI run markers without matching a longer run ID', async () => { + const api = fakeGithub([ + issue(248, 'CI/CD Pipeline Failed: Runner CI', { body: 'Failure\\n\\nworkflow-run-id:500\\nPlease investigate' }), + issue(249, 'CI/CD Pipeline Failed: Runner CI', { body: 'workflow-run-id:5010' }), + ]); + await report(api, 'Runner CI', 500); + assert.equal(api.issues.length, 2); + await report(api, 'Runner CI', 501); + assert.equal(api.issues.length, 3); + assert.match(api.issues[2].body, /workflow-run-id:501\n/); +}); + +// Untrusted or irrelevant workflow completions must not mutate repository issues. +test('ignores events outside the failure reporting boundary', async t => { + const cases = [ + ['successful run', { conclusion: 'success' }], + ['pull request', { event: 'pull_request' }], + ['fork run', { head_repository: { full_name: 'someone/runner' } }], + ['unknown workflow', { name: 'Other workflow' }], + ]; + for (const [name, overrides] of cases) { + await t.test(name, async () => { + const api = fakeGithub(); + await report(api, MAIN, 500, overrides); + assert.deepEqual(api.writes, []); + assert.equal(api.pages, 0); + }); + } +}); + +// API failures must surface to Actions instead of causing silent loss or duplicate creation. +test('propagates GitHub API errors', async t => { + for (const operation of ['listForRepo', 'getLabel', 'createLabel', 'create', 'update', 'addLabels']) { + await t.test(operation, async () => { + const existing = ['update', 'addLabels'].includes(operation); + const api = fakeGithub(existing ? [issue(248, 'Tracker', { body: TRACKER })] : []); + api.github.rest.issues[operation] = async () => { throw new Error('GitHub unavailable'); }; + await assert.rejects(report(api, MAIN, 500), /GitHub unavailable/); + assert.equal(api.issues.length, existing ? 1 : 0); + }); + } +}); + +function issue(number, title, overrides = {}) { + return { number, title, body: '', state: 'open', labels: ['bug'], ...overrides }; +} + +function report(api, name, id, overrides = {}) { + return reportWorkflowFailure({ github: api.github, context: { + repo: { owner: 'canonical', repo: 'github-actions-runner' }, + payload: { workflow_run: { + name, id, run_attempt: 1, html_url: `https://github.com/canonical/github-actions-runner/actions/runs/${id}`, + conclusion: 'failure', event: 'schedule', + head_repository: { full_name: 'canonical/github-actions-runner' }, ...overrides, + } }, + } }); +} + +function fakeGithub(existing = []) { + const api = { issues: structuredClone(existing), writes: [], pages: 0, searches: [] }; + const labels = new Set(['bug']); + const issues = { + async listForRepo({ owner, repo, state, per_page, page = 1 }) { + assert.equal(`${owner}/${repo}`, 'canonical/github-actions-runner'); + assert.equal(state, 'all'); + api.pages++; + return { data: structuredClone(api.issues.slice((page - 1) * per_page, page * per_page)) }; + }, + async getLabel({ name }) { + if (!labels.has(name)) throw Object.assign(new Error('Not found'), { status: 404 }); + return { data: { name } }; + }, + async createLabel({ name }) { labels.add(name); return { data: { name } }; }, + async addLabels({ issue_number, labels: added }) { + const updated = api.issues.find(i => i.number === issue_number); + assert.ok(updated); + updated.labels.push(...added); + api.writes.push({ issue_number, labels: added }); + return { data: structuredClone(updated.labels) }; + }, + async create(fields) { + const created = issue(Math.max(0, ...api.issues.map(i => i.number)) + 1, fields.title, fields); + api.issues.push(created); + api.writes.push(fields); + return { data: structuredClone(created) }; + }, + async update({ issue_number, ...fields }) { + const updated = api.issues.find(i => i.number === issue_number); + assert.ok(updated); + Object.assign(updated, fields); + api.writes.push(fields); + return { data: structuredClone(updated) }; + }, + }; + const search = { + async issuesAndPullRequests({ q, per_page }) { + api.searches.push(q); + const marker = q.match(/"(workflow-run-id:\d+)"/)[1]; + const matches = api.issues.filter(issue => issue.state === 'open' && !issue.pull_request && + (issue.body || '').includes(marker)); + return { data: { total_count: matches.length, incomplete_results: false, + items: structuredClone(matches.slice(0, per_page)) } }; + }, + }; + api.github = { rest: { issues, search }, async paginate(method, params) { + const result = []; + for (let page = 1; ; page++) { + const { data } = await method({ ...params, page }); + result.push(...data); + if (data.length < params.per_page) return result; + } + } }; + return api; +} diff --git a/.github/workflows/create-issue-on-failure.yaml b/.github/workflows/create-issue-on-failure.yaml index a29dbe0e51a..cd204a48cb6 100644 --- a/.github/workflows/create-issue-on-failure.yaml +++ b/.github/workflows/create-issue-on-failure.yaml @@ -16,6 +16,12 @@ permissions: issues: write contents: read +# Serialize shared issue writes without replacing pending failure reports. +concurrency: + group: workflow-failure-reporting + cancel-in-progress: false + queue: max + jobs: create-issue: # Only run this job if the triggered workflow concluded with a failure @@ -24,34 +30,12 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v7 + with: + persist-credentials: false - - name: Create GitHub Issue on Failure - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WORKFLOW_NAME: ${{ github.event.workflow_run.name }} - RUN_URL: ${{ github.event.workflow_run.html_url }} - REPOSITORY: ${{ github.repository }} - RUN_ID: ${{ github.event.workflow_run.id }} - run: | - ISSUE_TITLE="CI/CD Pipeline Failed: ${WORKFLOW_NAME}" - ISSUE_MARKER="workflow-run-id:${RUN_ID}" - ISSUE_BODY=$'The GitHub Actions workflow **'"${WORKFLOW_NAME}"'** has failed.\n\n**Run URL:** '"${RUN_URL}"'\n**Run ID:** '"${RUN_ID}"'\n\n'"${ISSUE_MARKER}"$'\n\nPlease investigate and resolve the issue.' - - # Check if an issue for this specific workflow run already exists to prevent duplicate spam on retries - EXISTING_ISSUE=$(gh issue list --repo "${REPOSITORY}" --state open --search "\"${ISSUE_MARKER}\" in:body" --json number --jq '.[0].number // empty') - if [ -n "$EXISTING_ISSUE" ]; then - echo "An issue for this workflow run already exists (#${EXISTING_ISSUE}). Skipping creation." - else - # gh issue create fails outright if a label is missing, so (re)create it first - gh label create "ci-failure" \ - --repo "${REPOSITORY}" \ - --description "Automated CI/CD pipeline failure" \ - --color "B60205" \ - --force - gh issue create \ - --repo "${REPOSITORY}" \ - --title "${ISSUE_TITLE}" \ - --body "${ISSUE_BODY}" \ - --label "bug" \ - --label "ci-failure" - fi + - name: Report workflow failure + uses: actions/github-script@v8 + with: + script: | + const { reportWorkflowFailure } = require('./.github/scripts/report-workflow-failure.cjs'); + await reportWorkflowFailure({ github, context }); diff --git a/.github/workflows/test-failure-reporting.yaml b/.github/workflows/test-failure-reporting.yaml new file mode 100644 index 00000000000..4922741ed10 --- /dev/null +++ b/.github/workflows/test-failure-reporting.yaml @@ -0,0 +1,29 @@ +name: Test workflow failure reporting + +on: + pull_request: + paths: + - .github/scripts/report-workflow-failure* + - .github/workflows/create-issue-on-failure.yaml + - .github/workflows/test-failure-reporting.yaml + push: + branches: [main] + paths: + - .github/scripts/report-workflow-failure* + - .github/workflows/create-issue-on-failure.yaml + - .github/workflows/test-failure-reporting.yaml + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: '24' + - run: node --test .github/scripts/report-workflow-failure.test.cjs