diff --git a/.config/mise/lib/git.sh b/.config/mise/lib/git.sh new file mode 100644 index 00000000000..0656c618068 --- /dev/null +++ b/.config/mise/lib/git.sh @@ -0,0 +1,12 @@ +# shellcheck shell=bash +# git_push_as pushes with the given token instead of +# the job's GITHUB_TOKEN. The token is read from the environment by the +# credential helper, so it never appears on a command line. actions/checkout +# sends GITHUB_TOKEN as an extra header, which would win over the helper, so +# that header is reset for this push. +# shellcheck disable=SC2016 # expanded by the credential helper, not here +git_push_as() { + PUSH_TOKEN="$1" git -c http.https://github.com/.extraheader= -c credential.helper= \ + -c 'credential.helper=!f() { echo username=x-access-token; echo "password=${PUSH_TOKEN}"; }; f' \ + push "${@:2}" +} diff --git a/.config/mise/tasks/github/workflows/trogonstack-sync/fetch-merge b/.config/mise/tasks/github/workflows/trogonstack-sync/fetch-merge new file mode 100755 index 00000000000..59d46347dee --- /dev/null +++ b/.config/mise/tasks/github/workflows/trogonstack-sync/fetch-merge @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +#MISE description="Fetch the bundled merge and check it is the commit the merge job recorded" +set -euo pipefail +# The sha was recorded before any upstream code ran, so a test that replaced +# the bundle is caught here. +git fetch --no-tags "$RUNNER_TEMP/sync.bundle" HEAD +if [ "$(git rev-parse FETCH_HEAD)" != "$MERGE_SHA" ]; then + echo "The bundle holds $(git rev-parse FETCH_HEAD), not $MERGE_SHA" >&2 + exit 1 +fi diff --git a/.config/mise/tasks/github/workflows/trogonstack-sync/merge-upstream b/.config/mise/tasks/github/workflows/trogonstack-sync/merge-upstream index e0526daf8b9..ab3ba6ff0e5 100755 --- a/.config/mise/tasks/github/workflows/trogonstack-sync/merge-upstream +++ b/.config/mise/tasks/github/workflows/trogonstack-sync/merge-upstream @@ -1,33 +1,22 @@ #!/usr/bin/env bash -#MISE description="Merge the upstream tag, or open a conflict issue and fail" +#MISE description="Merge the upstream tag, committing any conflicts so they can be published as a pull request" set -euo pipefail git config user.name 'github-actions[bot]' git config user.email '41898282+github-actions[bot]@users.noreply.github.com' if git merge --no-ff --no-edit -m "chore(sync): merge upstream $TAG" "refs/tags/$TAG"; then - echo "conflict=false" >> "$GITHUB_OUTPUT" - exit 0 -fi -echo "conflict=true" >> "$GITHUB_OUTPUT" -git diff --name-only --diff-filter=U > "$RUNNER_TEMP/conflicts.txt" -git merge --abort -title="Upstream sync conflict: $TAG" -{ - echo "Merging upstream $TAG into main hit conflicts." - echo - echo "Conflicting files:" - sed 's/^/- /' "$RUNNER_TEMP/conflicts.txt" - echo - echo "Run: $RUN_URL" - echo - echo "Resolve locally with: git merge $TAG, then push the result to main." -} > "$RUNNER_TEMP/conflict-body.txt" -# gh under GITHUB_TOKEN is non-interactive: pass the body by file so no -# editor or prompt is needed, and skip labels that may not exist. -existing="$(gh issue list --state open --limit 200 --json number,title \ - --jq "[.[] | select(.title == \"$title\")] | length")" -if [ "$existing" = "0" ]; then - gh issue create --title "$title" --body-file "$RUNNER_TEMP/conflict-body.txt" + conflict=false else - echo "Open conflict issue already exists" + # Commit the conflict markers instead of aborting, so the merge can be + # finished on the pull request branch rather than redone by hand. + git diff --name-only --diff-filter=U > "$RUNNER_TEMP/conflicts.txt" + git add -A + git commit --no-edit -m "chore(sync): merge upstream $TAG with conflicts" + conflict=true fi -exit 1 +# The publish job pushes exactly this commit, so it is bundled here, before +# the build and tests run any upstream code. +git bundle create "$RUNNER_TEMP/sync.bundle" HEAD ^origin/main +{ + echo "conflict=$conflict" + echo "sha=$(git rev-parse HEAD)" +} >> "$GITHUB_OUTPUT" diff --git a/.config/mise/tasks/github/workflows/trogonstack-sync/publish-pull-request b/.config/mise/tasks/github/workflows/trogonstack-sync/publish-pull-request index 03369b7ae69..b9e72d2584d 100755 --- a/.config/mise/tasks/github/workflows/trogonstack-sync/publish-pull-request +++ b/.config/mise/tasks/github/workflows/trogonstack-sync/publish-pull-request @@ -1,22 +1,44 @@ #!/usr/bin/env bash -#MISE description="Publish a merge that fails verification as a pull request and fail" +#MISE description="Publish a merge that cannot land on main as a draft pull request and fail" set -euo pipefail +source "$MISE_PROJECT_ROOT/.config/mise/lib/git.sh" + branch="sync/$TAG" -git fetch origin "$branch" || true -git push --force-with-lease origin "HEAD:refs/heads/$branch" -title="chore(sync): merge upstream $TAG" +# An open pull request may already carry fixes, so leave its branch alone. +existing="$(GH_TOKEN="$BARNACLE_TOKEN" gh pr list --state open --head "$branch" --json url --jq '.[0].url // ""')" +if [ -n "$existing" ]; then + echo "Upstream $TAG is still pending on $existing" >> "$GITHUB_STEP_SUMMARY" + exit 1 +fi + +if [ "$CONFLICT" = "true" ]; then + reason="it conflicts with main." +elif [ "$VERIFIED" = "failure" ]; then + reason="build or tests failed." +else + reason="$PUSH_REASON" +fi + +# Pushed as Barnacle so the pull request runs CI and may change workflows. +# Without an open pull request the branch is stale, so it is replaced. +git_push_as "$BARNACLE_TOKEN" --force origin "$MERGE_SHA:refs/heads/$branch" + { - echo "Merging upstream $TAG into main is clean but build or tests failed." + echo "Merging upstream $TAG into main needs a fix before it can land: $reason" + # Only a conflicting merge skips the build and tests, so only then is the + # list known to be written before any upstream code ran. + if [ "$CONFLICT" = "true" ] && [ -s "$RUNNER_TEMP/conflicts.txt" ]; then + echo + echo "Conflicting files, committed with their markers:" + # shellcheck disable=SC2016 # literal backticks for markdown + head -n 200 "$RUNNER_TEMP/conflicts.txt" | sed 's/^/- `/; s/$/`/' + fi echo - echo "Failed run: $RUN_URL" + echo "Run: $RUN_URL" echo - echo "Fix on this branch, then merge it into main." + echo "Fix on this branch, mark it ready, then merge it into main." } > "$RUNNER_TEMP/pr-body.txt" -existing="$(gh pr list --state open --head "$branch" --json number --jq 'length')" -if [ "$existing" = "0" ]; then - gh pr create --base main --head "$branch" --title "$title" \ - --body-file "$RUNNER_TEMP/pr-body.txt" -else - echo "Open sync PR already exists" -fi +url="$(GH_TOKEN="$BARNACLE_TOKEN" gh pr create --draft --base main --head "$branch" \ + --title "chore(sync): merge upstream $TAG" --body-file "$RUNNER_TEMP/pr-body.txt")" +echo "Upstream $TAG is pending on $url" >> "$GITHUB_STEP_SUMMARY" exit 1 diff --git a/.config/mise/tasks/github/workflows/trogonstack-sync/push-main b/.config/mise/tasks/github/workflows/trogonstack-sync/push-main index 09f9f7b5b7f..5188bdbfe45 100755 --- a/.config/mise/tasks/github/workflows/trogonstack-sync/push-main +++ b/.config/mise/tasks/github/workflows/trogonstack-sync/push-main @@ -1,26 +1,23 @@ #!/usr/bin/env bash -#MISE description="Push the verified merge to main and dispatch the image build" +#MISE description="Push the verified merge to main and dispatch the image build, or record why it needs a pull request" set -euo pipefail -# GITHUB_TOKEN cannot push commits that touch .github/workflows, and -# upstream changes those every month or so. Say so in an issue -# instead of leaving a bare red run. -if ! git push origin HEAD:main; then - title="Upstream sync needs a manual push: $TAG" - if git diff --quiet HEAD^1 HEAD -- .github/workflows; then - reason="The push to main was rejected." - else - reason="The merge changes .github/workflows, which GITHUB_TOKEN may not push." - fi - printf '%s\n\nRun: %s\n\nPush it locally with: git merge %s, then git push origin main.\n' \ - "$reason" "$RUN_URL" "$TAG" > "$RUNNER_TEMP/push-body.txt" - existing="$(gh issue list --state open --limit 200 --json title \ - --jq "[.[] | select(.title == \"$title\")] | length")" - if [ "$existing" = "0" ]; then - gh issue create --title "$title" --body-file "$RUNNER_TEMP/push-body.txt" - fi - exit 1 +# The publish job runs these tasks from main with the Barnacle key in reach, so +# a merge that changes them, mise configuration or a local action must land +# through a reviewed pull request, never this push. GITHUB_TOKEN already may +# not push .github/workflows. +trusted='^(\.github/|\.config/|\.?mise/|\.?mise(\.[^/]+)?\.toml$|\.tool-versions$)' +reason="" +# Captured first so a failing diff stops the script instead of reading as no +# match, and so grep -q cannot cut the pipe short under pipefail. +changed="$(git diff --name-only --no-renames HEAD "$MERGE_SHA")" +if grep -Eq "$trusted" <<<"$changed"; then + reason="the merge changes .github, .config or mise configuration, which the publish job runs." +elif git push origin "$MERGE_SHA:refs/heads/main"; then + # Pushes made with GITHUB_TOKEN do not trigger workflows, but + # workflow_dispatch does, so start the image build explicitly. + gh workflow run trogonstack-image.yml --ref main -f release=true + echo "Merged upstream $TAG into main and dispatched the image build with release" >> "$GITHUB_STEP_SUMMARY" +else + reason="the push to main was rejected." fi -# Pushes made with GITHUB_TOKEN do not trigger workflows, but -# workflow_dispatch does, so start the image build explicitly. -gh workflow run trogonstack-image.yml --ref main -f release=true -echo "Merged upstream $TAG into main and dispatched the image build with release" >> "$GITHUB_STEP_SUMMARY" +echo "reason=$reason" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/trogonstack-sync.yml b/.github/workflows/trogonstack-sync.yml index ac774d13147..b4549d74f4f 100644 --- a/.github/workflows/trogonstack-sync.yml +++ b/.github/workflows/trogonstack-sync.yml @@ -11,12 +11,7 @@ on: - cron: '17 6 * * 5' workflow_dispatch: -permissions: - contents: write - pull-requests: write - issues: write - actions: write - packages: read +permissions: {} concurrency: group: trogonstack-sync @@ -29,14 +24,26 @@ env: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} jobs: - sync: + # Builds and tests upstream code, so it can read the repository and nothing + # else. The merge is bundled before any of that code runs, and publish + # pushes it by sha, so a test cannot swap the commit that lands. + merge: runs-on: ubuntu-latest timeout-minutes: 60 + permissions: + contents: read + outputs: + tag: ${{ steps.tag.outputs.tag }} + up_to_date: ${{ steps.tag.outputs.up_to_date }} + sha: ${{ steps.merge.outputs.sha }} + conflict: ${{ steps.merge.outputs.conflict }} + verified: ${{ steps.verify.outcome }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main fetch-depth: 0 + persist-credentials: false - uses: ./.github/actions/setup-mise @@ -49,35 +56,128 @@ jobs: env: TAG: ${{ steps.tag.outputs.tag }} run: mise run github:workflows:trogonstack-sync:merge-upstream - - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: steps.tag.outputs.up_to_date == 'false' + with: + name: sync + path: | + ${{ runner.temp }}/sync.bundle + ${{ runner.temp }}/conflicts.txt + retention-days: 7 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + if: steps.tag.outputs.up_to_date == 'false' && steps.merge.outputs.conflict == 'false' with: go-version-file: go.mod - cache: true + # A cache saved after upstream tests run would be restored by + # later builds on main. + cache: false # The full suite is run as is. If upstream later adds tests that need # credentials or network not available in CI, narrow this to a package # subset here instead of skipping verification. - name: Build and test id: verify - if: steps.tag.outputs.up_to_date == 'false' + if: steps.tag.outputs.up_to_date == 'false' && steps.merge.outputs.conflict == 'false' continue-on-error: true run: mise run github:workflows:trogonstack-sync:verify - - name: Publish failing merge as a pull request - if: steps.tag.outputs.up_to_date == 'false' && steps.verify.outcome == 'failure' - env: - TAG: ${{ steps.tag.outputs.tag }} - run: mise run github:workflows:trogonstack-sync:publish-pull-request + + # The only job that can read the Barnacle key: the environment admits it + # from main, and GCP admits only this file. It runs mise tasks and a local + # action from main, so push-main never lands a merge that changes them; + # those, like workflow changes GITHUB_TOKEN may not push, go through a + # reviewed pull request. + # Barnacle opens the sync pull requests, so they run CI and may change + # workflows, which GITHUB_TOKEN can do neither of. + publish: + needs: merge + if: needs.merge.outputs.up_to_date == 'false' + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: github-app-barnacle + permissions: + contents: write + actions: write + id-token: write + env: + TAG: ${{ needs.merge.outputs.tag }} + MERGE_SHA: ${{ needs.merge.outputs.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: sync + path: ${{ runner.temp }} + + - uses: ./.github/actions/setup-mise + + - name: Fetch the merge + run: mise run github:workflows:trogonstack-sync:fetch-merge + - name: Push main and build image - if: steps.tag.outputs.up_to_date == 'false' && steps.verify.outcome == 'success' - env: - TAG: ${{ steps.tag.outputs.tag }} + id: push + if: needs.merge.outputs.verified == 'success' run: mise run github:workflows:trogonstack-sync:push-main - # Main may hold fork-only commits that were never released. The release - # workflow is idempotent, so run it for HEAD when it has an image, and - # build and release that commit when it has none. + + - name: Google Cloud auth + if: needs.merge.outputs.verified != 'success' || steps.push.outputs.reason != '' + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3 + with: + workload_identity_provider: ${{ vars.BARNACLE_WORKLOAD_IDENTITY_PROVIDER }} + + - name: Barnacle key + id: barnacle_key + if: needs.merge.outputs.verified != 'success' || steps.push.outputs.reason != '' + uses: google-github-actions/get-secretmanager-secrets@bc9c54b29fdffb8a47776820a7d26e77b379d262 # v3.0.0 + with: + secrets: |- + private_key:${{ vars.BARNACLE_PRIVATE_KEY_SECRET }} + + - name: Barnacle token + id: barnacle + if: needs.merge.outputs.verified != 'success' || steps.push.outputs.reason != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.BARNACLE_APP_CLIENT_ID }} + private-key: ${{ steps.barnacle_key.outputs.private_key }} + repositories: ${{ github.event.repository.name }} + permission-contents: write + permission-pull-requests: write + permission-workflows: write + + - name: Publish the merge as a pull request + if: needs.merge.outputs.verified != 'success' || steps.push.outputs.reason != '' + env: + BARNACLE_TOKEN: ${{ steps.barnacle.outputs.token }} + CONFLICT: ${{ needs.merge.outputs.conflict }} + VERIFIED: ${{ needs.merge.outputs.verified }} + PUSH_REASON: ${{ steps.push.outputs.reason }} + run: mise run github:workflows:trogonstack-sync:publish-pull-request + + # Main may hold fork-only commits that were never released. The release + # workflow is idempotent, so run it for HEAD when it has an image, and build + # and release that commit when it has none. + release: + needs: merge + if: needs.merge.outputs.up_to_date == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + actions: write + packages: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + + - uses: ./.github/actions/setup-mise + - name: Release current main - if: steps.tag.outputs.up_to_date == 'true' env: REGISTRY_TOKEN: ${{ github.token }} ACTOR: ${{ github.actor }}