Repository navigation
feat: Verify Bugbot findings against repository evidence before publication (#950) #951
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
80407b7
6128fdd
f8de480
9150e57
0b984e8
1c85dd5
5bf303f
6b2ddc0
864bbba
19da061
6e0b61e
a3f7085
a4c8ca9
5016d84
f2dca44
9821435
e477e0e
9a6a742
1e12133
f1b7821
7d854c3
232b8eb
2d1d92c
580811b
9b41e7d
d102515
e99e87e
77eb5df
5aed48f
c271981
4fe9a40
55124fb
fdae8c2
0d92082
fa14c19
c48a934
d161615
be36415
2799f53
d8fad2e
6337c10
9debb2e
6e22582
d7bb6a0
c176fc3
1fd145f
f3f9b76
c45ac54
10c9e80
98654c7
d78caa7
4bfdb03
d7b16c6
24d4c3e
737e938
17f7e6f
c66403a
7e98fdd
d077c9e
cc36775
8866fa5
335ffe4
9279140
d497a64
ee70cb4
2cde9f3
5f38bf9
57533f2
4852832
f5d8232
b180ee4
be25127
4597dd0
b4516db
fd3d84d
f00501d
f786cfd
b12d0e9
13f999a
b875993
a539e7f
94843e7
8a10824
c23a667
fac1e13
4b052fb
33769e3
39cd159
38b13dc
8b32586
3050811
cb249e4
b6f2511
8cb91ea
53b96fe
53b2173
c7ef155
7ead27b
2827470
f7f72f9
b0d181f
502e7d7
d03154b
84211d4
03f8e6e
9e9f153
23ca43c
16d18f2
542a527
ca75978
d26d0d6
19bf6f3
86e2430
b87c793
46d1d7f
6a7121e
ea55a97
069a10b
b16e6ca
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,8 +5,6 @@ on: | |
| branches: [master] | ||
| pull_request: | ||
| types: [opened, synchronize] | ||
| merge_group: | ||
| types: [checks_requested] | ||
| workflow_dispatch: | ||
|
|
||
| concurrency: | ||
|
|
@@ -17,57 +15,122 @@ defaults: | |
| run: | ||
| shell: bash | ||
|
|
||
| permissions: | ||
| contents: read | ||
|
|
||
| jobs: | ||
| ci-check: | ||
| name: CI Check | ||
| # A merge group can contain fork code without exposing the originating PR here. | ||
| runs-on: ${{ fromJSON((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository)) && '["ubuntu-latest"]' || '["self-hosted","codex","Linux"]') }} | ||
| timeout-minutes: 20 | ||
| permissions: | ||
| contents: read | ||
| static-checks: | ||
| name: CI / Static checks | ||
| if: &source-ci ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} | ||
| runs-on: &source-runner | ||
| group: vypdev | ||
| labels: [self-hosted, codex, Linux] | ||
| timeout-minutes: 10 | ||
| steps: | ||
| - name: Verify Ubuntu runner | ||
| - &verify-ubuntu | ||
| name: Verify Ubuntu runner | ||
| run: . /etc/os-release && test "$ID" = ubuntu | ||
| - uses: actions/checkout@v5 | ||
|
|
||
| - &checkout | ||
| uses: actions/checkout@v5 | ||
| with: | ||
| persist-credentials: false | ||
| # Contract tests inspect the immutable in-repository action revision | ||
| # used by distributed workflows, so its commit must be available. | ||
| fetch-depth: 0 | ||
|
|
||
| - name: Set up pnpm | ||
| # Avoid the standalone @pnpm/exe binary, which has no working Intel | ||
| # macOS build for this pnpm line. | ||
| - name: Validate Git diff | ||
| env: | ||
| EVENT_NAME: ${{ github.event_name }} | ||
| PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} | ||
| PUSH_BEFORE_SHA: ${{ github.event.before }} | ||
| run: | | ||
| set -euo pipefail | ||
| case "$EVENT_NAME" in | ||
| pull_request) base="$PR_BASE_SHA" ;; | ||
| push) base="$PUSH_BEFORE_SHA" ;; | ||
| workflow_dispatch) base="$(git rev-parse HEAD^)" ;; | ||
| *) echo "Unsupported diff-check event" >&2; exit 1 ;; | ||
| esac | ||
| if [[ "$EVENT_NAME" == push && "$base" =~ ^0{40}$ ]]; then | ||
| base="$(git hash-object -t tree --stdin </dev/null)" | ||
| fi | ||
| [[ "$base" =~ ^[0-9a-f]{40}$ ]] || { echo "Missing diff-check base commit" >&2; exit 1; } | ||
| git -c core.whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol diff --check "$base" HEAD -- . ':(exclude)build/**' | ||
|
|
||
| - &pnpm | ||
| name: Set up pnpm | ||
| uses: pnpm/action-setup@v5 | ||
| with: | ||
| version: 10.12.4 | ||
| standalone: false | ||
| dest: ${{ runner.temp }}/setup-pnpm-${{ github.run_id }}-${{ github.run_attempt }}-${{ github.job }} | ||
|
|
||
| - name: Set up Node.js 24 | ||
| - &node | ||
| name: Set up Node.js 24 | ||
| uses: actions/setup-node@v7 | ||
| with: | ||
| node-version: '24.x' | ||
| cache: pnpm | ||
| # Each self-hosted runner can reuse its local pnpm store. | ||
| package-manager-cache: false | ||
|
|
||
| - &runner-scratch | ||
| name: Use runner scratch for CI | ||
| # Self-hosted runners can mount /tmp on a smaller filesystem than the work directory. | ||
| run: printf 'TMPDIR=%s\n' "$RUNNER_TEMP" >> "$GITHUB_ENV" | ||
|
|
||
| - name: Install dependencies | ||
| - &install | ||
| name: Install dependencies | ||
| run: pnpm install --frozen-lockfile | ||
|
|
||
| - name: Validate generated guidance checkout | ||
| run: | | ||
| git ls-files --eol -- .copilot/repository-profile.json .copilot/AGENT_GUIDE.md .agents/skills/copilot-repository-workflow/SKILL.md | ||
| pnpm run validate:agent-docs | ||
| - name: Lint | ||
| run: pnpm run lint | ||
|
|
||
| - name: Typecheck | ||
| run: pnpm run typecheck | ||
|
|
||
| - name: Validate documentation assets | ||
| run: pnpm run validate:docs-page | ||
|
|
||
| - name: Validate documentation contract | ||
| run: pnpm run validate:documentation | ||
|
|
||
| - name: Validate workflow contract | ||
| run: pnpm run validate:workflows | ||
|
|
||
| - name: Validate product specification catalog | ||
| run: pnpm run validate:specifications | ||
|
|
||
| - name: Validate setup acceptance ledger | ||
| run: pnpm run validate:setup-acceptance | ||
|
|
||
| - name: Audit production dependencies | ||
| run: pnpm audit --prod --audit-level=high | ||
|
|
||
| - name: Validate Bugbot quality benchmark | ||
| run: pnpm run eval:bugbot | ||
|
|
||
| - name: Validate Bugbot shadow verification | ||
| run: pnpm run test:bugbot-shadow-verification | ||
|
|
||
| build-package: | ||
| name: CI / Build and package | ||
| if: *source-ci | ||
| runs-on: *source-runner | ||
| timeout-minutes: 10 | ||
| steps: | ||
| - *verify-ubuntu | ||
| - *checkout | ||
| - *pnpm | ||
| - *node | ||
| - *runner-scratch | ||
| - *install | ||
|
|
||
| - name: Build | ||
| run: pnpm run build | ||
|
|
||
| # Checked-in ncc bundles are generated on Unix; Windows output differs. | ||
| - name: Validate generated bundles | ||
| if: runner.os != 'Windows' | ||
| run: pnpm run validate:build | ||
|
|
||
| - name: Validate extension architecture and reproducibility | ||
|
|
@@ -79,19 +142,33 @@ jobs: | |
| - name: Smoke test npm package | ||
| run: pnpm run smoke:npm-package | ||
|
|
||
| - name: Typecheck | ||
| run: pnpm run typecheck | ||
| coverage: | ||
| name: CI / Tests and coverage | ||
| if: *source-ci | ||
| runs-on: *source-runner | ||
| timeout-minutes: 25 | ||
| steps: | ||
| - *verify-ubuntu | ||
| - *checkout | ||
| - *pnpm | ||
| - *node | ||
| - *runner-scratch | ||
| - *install | ||
|
|
||
| - name: Validate generated guidance checkout | ||
| run: | | ||
| git ls-files --eol -- .copilot/repository-profile.json .copilot/AGENT_GUIDE.md .agents/skills/copilot-repository-workflow/SKILL.md | ||
| pnpm run validate:agent-docs | ||
|
|
||
| - name: Run tests with coverage | ||
| run: pnpm run test:coverage | ||
|
|
||
| - name: Validate Bugbot quality benchmark | ||
| run: pnpm run eval:bugbot | ||
| - name: Verify tests preserved generated guidance | ||
| run: pnpm run validate:agent-docs | ||
|
|
||
| - name: Upload coverage to Codecov | ||
| uses: codecov/codecov-action@v6 | ||
| # Coverage reporting is advisory; Codecov v6 requires gpg even when | ||
| # fail_ci_if_error is false, and the internal macOS runner may not have it. | ||
| # Coverage reporting is advisory; the local coverage gates above are required. | ||
| continue-on-error: true | ||
| with: | ||
| files: ./coverage/lcov.info,./extension/coverage/lcov.info | ||
|
|
@@ -100,44 +177,23 @@ jobs: | |
| token: ${{ secrets.CODECOV_TOKEN }} | ||
| verbose: true | ||
|
|
||
| - name: Lint | ||
| run: pnpm run lint | ||
|
|
||
| - name: Validate documentation | ||
| run: pnpm run validate:agent-docs | ||
|
|
||
| - name: Validate documentation assets | ||
| run: pnpm run validate:docs-page | ||
|
|
||
| - name: Validate documentation contract | ||
| run: pnpm run validate:documentation | ||
|
|
||
| - name: Validate workflow contract | ||
| run: pnpm run validate:workflows | ||
|
|
||
| - name: Validate product specification catalog | ||
| run: pnpm run validate:specifications | ||
|
|
||
| - name: Validate setup acceptance ledger | ||
| run: pnpm run validate:setup-acceptance | ||
|
|
||
| - name: Validate Git diff | ||
| ci-check: | ||
| # Keep the ruleset and approval consumer bound to this exact check name. | ||
| name: CI Check | ||
| if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) }} | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fork pull requests can bypass the required CI gatesSeverity: medium Category: security · Confidence: 97% Location: For fork pull requests, the shared Suggested fix: |
||
| needs: [static-checks, build-package, coverage] | ||
| runs-on: *source-runner | ||
| timeout-minutes: 2 | ||
| permissions: {} | ||
| steps: | ||
| - name: Require all CI jobs to succeed | ||
| env: | ||
| EVENT_NAME: ${{ github.event_name }} | ||
| PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} | ||
| MERGE_BASE_SHA: ${{ github.event.merge_group.base_sha }} | ||
| PUSH_BEFORE_SHA: ${{ github.event.before }} | ||
| STATIC_RESULT: ${{ needs.static-checks.result }} | ||
| BUILD_RESULT: ${{ needs.build-package.result }} | ||
| COVERAGE_RESULT: ${{ needs.coverage.result }} | ||
| run: | | ||
| set -euo pipefail | ||
| case "$EVENT_NAME" in | ||
| pull_request) base="$PR_BASE_SHA" ;; | ||
| merge_group) base="$MERGE_BASE_SHA" ;; | ||
| push) base="$PUSH_BEFORE_SHA" ;; | ||
| workflow_dispatch) base="$(git rev-parse HEAD^)" ;; | ||
| *) echo "Unsupported diff-check event" >&2; exit 1 ;; | ||
| esac | ||
| if [[ "$EVENT_NAME" == push && "$base" =~ ^0{40}$ ]]; then | ||
| base="$(git hash-object -t tree --stdin </dev/null)" | ||
| fi | ||
| [[ "$base" =~ ^[0-9a-f]{40}$ ]] || { echo "Missing diff-check base commit" >&2; exit 1; } | ||
| git -c core.whitespace=blank-at-eol,blank-at-eof,space-before-tab,cr-at-eol diff --check "$base" HEAD -- . ':(exclude)build/**' | ||
| printf '%s\n' '| Job | Result |' '| --- | --- |' \ | ||
| "| Static checks | $STATIC_RESULT |" \ | ||
| "| Build and package | $BUILD_RESULT |" \ | ||
| "| Tests and coverage | $COVERAGE_RESULT |" | tee -a "$GITHUB_STEP_SUMMARY" | ||
| [[ "$STATIC_RESULT" == success && "$BUILD_RESULT" == success && "$COVERAGE_RESULT" == success ]] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| name: Copilot - Bugbot Status Card | ||
| run-name: Bugbot status · PR ${{ inputs.pull_number }} | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| pull_number: | ||
| description: Pull request number | ||
| required: true | ||
| type: number | ||
| work: | ||
| description: Versioned publication handoff | ||
| required: true | ||
| type: string | ||
|
|
||
| jobs: | ||
| publish: | ||
| name: Publish Bugbot status card | ||
| if: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} | ||
| runs-on: [self-hosted, codex, Linux] | ||
| timeout-minutes: 5 | ||
| concurrency: | ||
| group: copilot-bugbot-status-${{ github.repository_id }}-${{ inputs.pull_number }} | ||
| queue: max | ||
| cancel-in-progress: false | ||
| permissions: | ||
| actions: read | ||
| checks: read | ||
| contents: read | ||
| issues: read | ||
| pull-requests: read | ||
| steps: | ||
| - uses: actions/checkout@v5 | ||
| with: | ||
| persist-credentials: false | ||
| - name: Publish canonical comment | ||
| uses: ./ | ||
| with: | ||
| token: ${{ secrets.PAT }} | ||
| pull-requests-locale: ${{ vars.PULL_REQUESTS_LOCALE || vars.REPOSITORY_LOCALE || 'en-US' }} | ||
| env: | ||
| COPILOT_BUGBOT_STATUS_CARD_WORKFLOW: 'write-v1' | ||
| COPILOT_EVIDENCE_TOKEN: ${{ github.token }} |
Uh oh!
There was an error while loading. Please reload this page.