From 045824a9be575c6d3f29afa6ccc6647d06163a73 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Mon, 10 Aug 2026 10:03:57 +0200 Subject: [PATCH 1/6] STAC-25565 Port the beest verification trigger to GitHub Actions beest_trigger_verification was the last job in the GitLab pipeline with no GitHub Actions equivalent. Every other job is covered by the STAC-25142 / STAC-25457 / STAC-25500 stack. In GitLab the job sits in the postbuild stage, needs both merge_docker_manifest jobs and is `when: manual`, passing AGENT_BRANCH_UNDER_TEST, AGENT_HASH_UNDER_TEST and TRIGGER_AGENT_X86_TESTS into the stackvista/integrations/beest project. beest has since migrated to GitHub, and its agent-x86.yml and arm.yml both expose workflow_dispatch with an agent_branch_under_test input, so the port is a cross-repo workflow dispatch rather than a pipeline trigger. beest resolves the agent image from the branch name, so the commit SHA is no longer part of its input contract; it is recorded in the run summary for traceability instead. Keeping the workflow workflow_dispatch-only preserves the GitLab `when: manual` semantics. These runs provision real EKS infrastructure in the sandbox account and share a single global concurrency lock in beest, so firing them automatically on push would queue runs behind each other and spend hours of cluster time per merge. The suite input defaults to x86, matching TRIGGER_AGENT_X86_TESTS: true; arm and both are available because beest now exposes an arm workflow that the GitLab job never reached. The scenario selector is passed through rather than re-declared, so beest stays the single owner of the valid scenario list. Requires a GitHub App credential in this repo with actions:write on StackVista/beest, provisioned via pulumi-infra: BEEST_DISPATCH_APP_CLIENT_ID (variable) and BEEST_DISPATCH_APP_PRIVATE_KEY (secret). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/beest-verification.yml | 109 +++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 .github/workflows/beest-verification.yml diff --git a/.github/workflows/beest-verification.yml b/.github/workflows/beest-verification.yml new file mode 100644 index 000000000000..7b48df3daa06 --- /dev/null +++ b/.github/workflows/beest-verification.yml @@ -0,0 +1,109 @@ +name: Beest verification + +on: + workflow_dispatch: + inputs: + suite: + description: Which beest agent suite to run (GitLab triggered x86 only) + type: choice + default: x86 + options: + - x86 + - arm + - both + agent_branch_under_test: + description: Agent branch beest should deploy (empty = the branch this workflow runs on) + type: string + default: "" + scenarios: + description: Beest scenario selector (empty = whatever the beest workflow defaults to) + type: string + default: "" + no_destroy: + description: Keep the beest infrastructure after the run (for debugging a failure) + type: boolean + default: false + beest_ref: + description: Ref of StackVista/beest to dispatch + type: string + default: main + +permissions: {} + +concurrency: + group: beest-verification-${{ github.ref }} + cancel-in-progress: false + +jobs: + trigger: + name: Trigger beest agent verification + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: {} + steps: + - name: Mint GitHub App token (dispatch beest workflows) + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.BEEST_DISPATCH_APP_CLIENT_ID }} + private-key: ${{ secrets.BEEST_DISPATCH_APP_PRIVATE_KEY }} + owner: StackVista + repositories: beest + permission-actions: write + + - name: Dispatch beest verification + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + SUITE: ${{ inputs.suite }} + AGENT_BRANCH: ${{ inputs.agent_branch_under_test || github.ref_name }} + AGENT_SHA: ${{ github.sha }} + SCENARIOS: ${{ inputs.scenarios }} + NO_DESTROY: ${{ inputs.no_destroy }} + BEEST_REF: ${{ inputs.beest_ref }} + run: | + set -euo pipefail + + case "${SUITE}" in + x86) workflows=("agent-x86.yml") ;; + arm) workflows=("arm.yml") ;; + both) workflows=("agent-x86.yml" "arm.yml") ;; + *) echo "::error::unknown suite '${SUITE}'"; exit 1 ;; + esac + + # beest resolves the agent image from the branch name; the SHA is recorded + # for traceability only and is not part of beest's dispatch contract. + echo "agent branch under test: ${AGENT_BRANCH} (${AGENT_SHA})" + + { + echo "## Beest verification dispatched" + echo + echo "| field | value |" + echo "| --- | --- |" + echo "| agent branch under test | \`${AGENT_BRANCH}\` |" + echo "| agent commit | \`${AGENT_SHA}\` |" + echo "| suite | \`${SUITE}\` |" + echo "| scenarios | \`${SCENARIOS:-(beest default)}\` |" + echo "| keep infrastructure | \`${NO_DESTROY}\` |" + echo "| beest ref | \`${BEEST_REF}\` |" + echo + } >> "${GITHUB_STEP_SUMMARY}" + + for wf in "${workflows[@]}"; do + args=(--repo StackVista/beest --ref "${BEEST_REF}") + args+=(--field "agent_branch_under_test=${AGENT_BRANCH}") + args+=(--field "no_destroy=${NO_DESTROY}") + if [ -n "${SCENARIOS}" ]; then + args+=(--field "scenarios=${SCENARIOS}") + fi + + echo "dispatching ${wf}" + gh workflow run "${wf}" "${args[@]}" + + # gh does not report the queued run, and every beest AWS workflow shares + # one global lock, so look the run up rather than assuming it started. + sleep 10 + url="$(gh run list --repo StackVista/beest --workflow "${wf}" \ + --limit 1 --json url --jq '.[0].url' 2>/dev/null || true)" + fallback="https://github.com/StackVista/beest/actions/workflows/${wf}" + echo "- \`${wf}\` -> ${url:-${fallback}}" >> "${GITHUB_STEP_SUMMARY}" + done From aa22627c8dcdb34cb449b99bed1e840caf5b4f5a Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Mon, 10 Aug 2026 10:17:28 +0200 Subject: [PATCH 2/6] STAC-25565: pin the agent commit when dispatching beest The first version of this workflow sent only the branch and put the SHA in the run summary, on the reasoning that beest had no hash input. That was the wrong conclusion: beest's own GitLab port dropped the input while keeping the machinery, so the missing input was a regression to fix rather than a constraint to design around. beest#61 restores it. A branch builds many images, so branch-only means always testing whichever build is newest -- there is no way to verify a specific commit or to reproduce a failure against the image that produced it. Defaults to this run's commit, matching the GitLab job's CI_COMMIT_SHA. When agent_branch_under_test points at some other branch our SHA does not exist there, so the pin is left unset and beest falls back instead of dispatching a hash that resolves to nothing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/beest-verification.yml | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/beest-verification.yml b/.github/workflows/beest-verification.yml index 7b48df3daa06..21c947bfd81a 100644 --- a/.github/workflows/beest-verification.yml +++ b/.github/workflows/beest-verification.yml @@ -15,6 +15,10 @@ on: description: Agent branch beest should deploy (empty = the branch this workflow runs on) type: string default: "" + agent_hash_under_test: + description: Agent commit beest should pin the image to (empty = the commit this workflow runs on) + type: string + default: "" scenarios: description: Beest scenario selector (empty = whatever the beest workflow defaults to) type: string @@ -56,6 +60,8 @@ jobs: GH_TOKEN: ${{ steps.app-token.outputs.token }} SUITE: ${{ inputs.suite }} AGENT_BRANCH: ${{ inputs.agent_branch_under_test || github.ref_name }} + AGENT_BRANCH_INPUT: ${{ inputs.agent_branch_under_test }} + AGENT_HASH_INPUT: ${{ inputs.agent_hash_under_test }} AGENT_SHA: ${{ github.sha }} SCENARIOS: ${{ inputs.scenarios }} NO_DESTROY: ${{ inputs.no_destroy }} @@ -70,9 +76,17 @@ jobs: *) echo "::error::unknown suite '${SUITE}'"; exit 1 ;; esac - # beest resolves the agent image from the branch name; the SHA is recorded - # for traceability only and is not part of beest's dispatch contract. - echo "agent branch under test: ${AGENT_BRANCH} (${AGENT_SHA})" + # A branch builds many images, so pin the exact commit rather than letting + # beest pick the newest build. Default to this run's SHA, but only when the + # branch was not overridden -- against another branch our SHA means nothing, + # so leave it unset and let beest fall back. + agent_hash="${AGENT_HASH_INPUT}" + if [ -z "${agent_hash}" ] && [ -z "${AGENT_BRANCH_INPUT}" ]; then + agent_hash="${AGENT_SHA}" + fi + + echo "agent branch under test: ${AGENT_BRANCH}" + echo "agent commit under test: ${agent_hash:-(unpinned - beest chart default)}" { echo "## Beest verification dispatched" @@ -80,7 +94,7 @@ jobs: echo "| field | value |" echo "| --- | --- |" echo "| agent branch under test | \`${AGENT_BRANCH}\` |" - echo "| agent commit | \`${AGENT_SHA}\` |" + echo "| agent commit under test | \`${agent_hash:-(unpinned - beest chart default)}\` |" echo "| suite | \`${SUITE}\` |" echo "| scenarios | \`${SCENARIOS:-(beest default)}\` |" echo "| keep infrastructure | \`${NO_DESTROY}\` |" @@ -92,6 +106,9 @@ jobs: args=(--repo StackVista/beest --ref "${BEEST_REF}") args+=(--field "agent_branch_under_test=${AGENT_BRANCH}") args+=(--field "no_destroy=${NO_DESTROY}") + if [ -n "${agent_hash}" ]; then + args+=(--field "agent_hash_under_test=${agent_hash}") + fi if [ -n "${SCENARIOS}" ]; then args+=(--field "scenarios=${SCENARIOS}") fi From 520f985b02fcf13fadd2f91644710699db782213 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 11 Aug 2026 10:00:47 +0200 Subject: [PATCH 3/6] STAC-25565 use the provisioned beest GitHub App credentials The workflow referenced BEEST_DISPATCH_APP_CLIENT_ID/PRIVATE_KEY, which are provisioned nowhere. The beest App already exists as BEEST_GH_APP_CLIENT_ID / BEEST_GH_APP_PRIVATE_KEY, matching the _GH_APP_* convention every other App credential in the estate follows. Those variables are currently bound only to the beest repo, so pulumi-infra must also bind them to stackstate-agent before this workflow can mint a token. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/beest-verification.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/beest-verification.yml b/.github/workflows/beest-verification.yml index 21c947bfd81a..ccea8754e43e 100644 --- a/.github/workflows/beest-verification.yml +++ b/.github/workflows/beest-verification.yml @@ -49,8 +49,8 @@ jobs: id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - client-id: ${{ vars.BEEST_DISPATCH_APP_CLIENT_ID }} - private-key: ${{ secrets.BEEST_DISPATCH_APP_PRIVATE_KEY }} + client-id: ${{ vars.BEEST_GH_APP_CLIENT_ID }} + private-key: ${{ secrets.BEEST_GH_APP_PRIVATE_KEY }} owner: StackVista repositories: beest permission-actions: write From 4ee159f36cb496f15ab0ef7b601bec9e4a28c475 Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Tue, 11 Aug 2026 10:13:57 +0200 Subject: [PATCH 4/6] STAC-25565 dispatch the consolidated beest pin field beest replaces agent_hash_under_test with hashes_under_test in StackVista/beest#63, so the dispatch has to send agent= through the new field. Must land together with that PR. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/beest-verification.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beest-verification.yml b/.github/workflows/beest-verification.yml index ccea8754e43e..b8b7e435e265 100644 --- a/.github/workflows/beest-verification.yml +++ b/.github/workflows/beest-verification.yml @@ -107,7 +107,7 @@ jobs: args+=(--field "agent_branch_under_test=${AGENT_BRANCH}") args+=(--field "no_destroy=${NO_DESTROY}") if [ -n "${agent_hash}" ]; then - args+=(--field "agent_hash_under_test=${agent_hash}") + args+=(--field "hashes_under_test=agent=${agent_hash}") fi if [ -n "${SCENARIOS}" ]; then args+=(--field "scenarios=${SCENARIOS}") From c58c65dca755f83bf2ba40d825f0c763b41e27bb Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Mon, 17 Aug 2026 15:19:14 +0200 Subject: [PATCH 5/6] STAC-25565 fail closed when the agent images are not published Review feedback on #444's follow-up: three ways the dispatch could report success while doing the wrong thing. Resolve the commit under test up front. beest only pins nodeAgent, clusterAgent and checksAgent to - when a hash reaches it, and its resolve-agent-hashes.sh deliberately refuses to resolve a branch ("Leave unset to use Helm chart defaults"). Passing agent_branch_under_test without a hash therefore deployed the chart default image, so the run could pass while testing an agent nobody selected. An overridden branch now resolves to its tip, and every dispatch carries a concrete commit. Require both multi-architecture manifest jobs to have succeeded for that commit before dispatching. The GitLab job was only playable once agent and cluster-agent had published; the standalone dispatch had no equivalent barrier and could spend a full beest run, holding the global beest AWS lock, on images that were never pushed. Select the dispatched run by diffing against the runs that existed beforehand. Taking the newest run raced with API propagation, concurrent dispatches and the shared lock, so the summary regularly linked an unrelated run. Needs actions:read and contents:read to resolve the commit and read its runs; the App token stays scoped to dispatching beest. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/beest-verification.yml | 148 ++++++++++++++++++----- 1 file changed, 118 insertions(+), 30 deletions(-) diff --git a/.github/workflows/beest-verification.yml b/.github/workflows/beest-verification.yml index b8b7e435e265..a79f6b62c002 100644 --- a/.github/workflows/beest-verification.yml +++ b/.github/workflows/beest-verification.yml @@ -16,7 +16,7 @@ on: type: string default: "" agent_hash_under_test: - description: Agent commit beest should pin the image to (empty = the commit this workflow runs on) + description: Agent commit beest should pin the images to (empty = the tip of the branch under test) type: string default: "" scenarios: @@ -42,9 +42,90 @@ jobs: trigger: name: Trigger beest agent verification runs-on: ubuntu-24.04 - timeout-minutes: 10 - permissions: {} + timeout-minutes: 15 + permissions: + actions: read + contents: read steps: + - name: Resolve the agent commit under test + id: resolve + env: + GH_TOKEN: ${{ github.token }} + AGENT_BRANCH: ${{ inputs.agent_branch_under_test || github.ref_name }} + AGENT_BRANCH_INPUT: ${{ inputs.agent_branch_under_test }} + AGENT_HASH_INPUT: ${{ inputs.agent_hash_under_test }} + AGENT_SHA: ${{ github.sha }} + run: | + set -euo pipefail + + # beest pins the agent, cluster-agent and checks-agent images to -, + # but only when a hash reaches it: resolve-agent-hashes.sh deliberately does not + # resolve a branch, so an unset hash silently falls back to the Helm chart default + # and the run passes while testing an agent nobody asked for. Always resolve to a + # concrete commit -- an overridden branch resolves to its tip. + if [ -n "${AGENT_HASH_INPUT}" ]; then + ref="${AGENT_HASH_INPUT}" + elif [ -n "${AGENT_BRANCH_INPUT}" ]; then + ref="${AGENT_BRANCH_INPUT}" + else + ref="${AGENT_SHA}" + fi + + if ! sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/${ref}" --jq .sha)"; then + echo "::error::cannot resolve '${ref}' to a commit in ${GITHUB_REPOSITORY}" + exit 1 + fi + + echo "agent branch under test: ${AGENT_BRANCH}" + echo "agent commit under test: ${sha}" + + { + echo "agent_branch=${AGENT_BRANCH}" + echo "agent_sha=${sha}" + } >> "${GITHUB_OUTPUT}" + + - name: Require published agent images for the commit under test + env: + GH_TOKEN: ${{ github.token }} + AGENT_SHA: ${{ steps.resolve.outputs.agent_sha }} + run: | + set -euo pipefail + + # In GitLab this job was only playable once both manifest jobs had published. + # A standalone dispatch has no such barrier, so without this check it can burn a + # full beest run -- and the global beest AWS lock -- on images that were never + # pushed, or whose publication failed. Fail closed. + required=( + "Publish and sign multi-architecture agent image" + "Publish and sign multi-architecture cluster-agent image" + ) + + succeeded="$( + gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${AGENT_SHA}&per_page=100" \ + --jq '.workflow_runs[].id' | + while read -r run_id; do + gh api --paginate \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100" \ + --jq '.jobs[] | select(.conclusion == "success") | .name' + done + )" + + missing=0 + for job in "${required[@]}"; do + if printf '%s\n' "${succeeded}" | grep -Fxq "${job}"; then + echo "published: ${job}" + else + echo "::error::no successful '${job}' for ${AGENT_SHA}; beest would deploy an image that does not exist" + missing=1 + fi + done + + if [ "${missing}" -ne 0 ]; then + echo "::error::agent images for ${AGENT_SHA} are not published - refusing to dispatch beest" + exit 1 + fi + - name: Mint GitHub App token (dispatch beest workflows) id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 @@ -59,10 +140,8 @@ jobs: env: GH_TOKEN: ${{ steps.app-token.outputs.token }} SUITE: ${{ inputs.suite }} - AGENT_BRANCH: ${{ inputs.agent_branch_under_test || github.ref_name }} - AGENT_BRANCH_INPUT: ${{ inputs.agent_branch_under_test }} - AGENT_HASH_INPUT: ${{ inputs.agent_hash_under_test }} - AGENT_SHA: ${{ github.sha }} + AGENT_BRANCH: ${{ steps.resolve.outputs.agent_branch }} + AGENT_HASH: ${{ steps.resolve.outputs.agent_sha }} SCENARIOS: ${{ inputs.scenarios }} NO_DESTROY: ${{ inputs.no_destroy }} BEEST_REF: ${{ inputs.beest_ref }} @@ -76,25 +155,13 @@ jobs: *) echo "::error::unknown suite '${SUITE}'"; exit 1 ;; esac - # A branch builds many images, so pin the exact commit rather than letting - # beest pick the newest build. Default to this run's SHA, but only when the - # branch was not overridden -- against another branch our SHA means nothing, - # so leave it unset and let beest fall back. - agent_hash="${AGENT_HASH_INPUT}" - if [ -z "${agent_hash}" ] && [ -z "${AGENT_BRANCH_INPUT}" ]; then - agent_hash="${AGENT_SHA}" - fi - - echo "agent branch under test: ${AGENT_BRANCH}" - echo "agent commit under test: ${agent_hash:-(unpinned - beest chart default)}" - { echo "## Beest verification dispatched" echo echo "| field | value |" echo "| --- | --- |" echo "| agent branch under test | \`${AGENT_BRANCH}\` |" - echo "| agent commit under test | \`${agent_hash:-(unpinned - beest chart default)}\` |" + echo "| agent commit under test | \`${AGENT_HASH}\` |" echo "| suite | \`${SUITE}\` |" echo "| scenarios | \`${SCENARIOS:-(beest default)}\` |" echo "| keep infrastructure | \`${NO_DESTROY}\` |" @@ -105,22 +172,43 @@ jobs: for wf in "${workflows[@]}"; do args=(--repo StackVista/beest --ref "${BEEST_REF}") args+=(--field "agent_branch_under_test=${AGENT_BRANCH}") + args+=(--field "hashes_under_test=agent=${AGENT_HASH}") args+=(--field "no_destroy=${NO_DESTROY}") - if [ -n "${agent_hash}" ]; then - args+=(--field "hashes_under_test=agent=${agent_hash}") - fi if [ -n "${SCENARIOS}" ]; then args+=(--field "scenarios=${SCENARIOS}") fi + # Snapshot the runs that already exist, because gh does not report the run it + # queued and every beest AWS workflow shares one global lock -- so the newest + # run is routinely somebody else's, or an older queued one. + before="$(gh run list --repo StackVista/beest --workflow "${wf}" \ + --branch "${BEEST_REF}" --limit 100 --json databaseId \ + --jq '[.[].databaseId | tostring] | join(",")' 2>/dev/null || true)" + echo "dispatching ${wf}" gh workflow run "${wf}" "${args[@]}" - # gh does not report the queued run, and every beest AWS workflow shares - # one global lock, so look the run up rather than assuming it started. - sleep 10 - url="$(gh run list --repo StackVista/beest --workflow "${wf}" \ - --limit 1 --json url --jq '.[0].url' 2>/dev/null || true)" - fallback="https://github.com/StackVista/beest/actions/workflows/${wf}" - echo "- \`${wf}\` -> ${url:-${fallback}}" >> "${GITHUB_STEP_SUMMARY}" + url="" + for _ in $(seq 1 30); do + sleep 5 + while read -r id candidate; do + case ",${before}," in + *",${id},"*) ;; + *) url="${candidate}"; break ;; + esac + done < <(gh run list --repo StackVista/beest --workflow "${wf}" \ + --branch "${BEEST_REF}" --limit 100 --json databaseId,url \ + --jq '.[] | "\(.databaseId) \(.url)"' 2>/dev/null || true) + if [ -n "${url}" ]; then + break + fi + done + + if [ -n "${url}" ]; then + echo "${wf} -> ${url}" + echo "- \`${wf}\` -> ${url}" >> "${GITHUB_STEP_SUMMARY}" + else + echo "::warning::dispatched ${wf} but no new run appeared within 150s" + echo "- \`${wf}\` -> https://github.com/StackVista/beest/actions/workflows/${wf} (new run not detected)" >> "${GITHUB_STEP_SUMMARY}" + fi done From bbc79578f156b11b0dfd38f2846306c137d61a7f Mon Sep 17 00:00:00 2001 From: Louis Parkin Date: Wed, 19 Aug 2026 09:56:09 +0200 Subject: [PATCH 6/6] STAC-25565 validate beest scenarios per suite and link runs unambiguously The scenario input was forwarded verbatim to every workflow the suite selects, but beest's choices are architecture-specific and disjoint: agent-x86.yml accepts contd-eks-x86-*, arm.yml accepts contd-eks-arm-*, and only "all" is common to both. With suite=both, any architecture-specific value therefore reached one workflow that rejects it. Validate the value against every selected workflow before dispatching any of them, so a rejected combination cannot leave one architecture running and holding the global beest AWS lock. The allowlist is duplicated from beest rather than read from it, which costs a bump here when beest gains a scenario. Reading it would need contents access to beest on the dispatch token and YAML parsing on the runner; failing closed with the valid values named is the cheaper trade. Run linking took the first run absent from the pre-dispatch snapshot, but a concurrent dispatch of the same workflow and ref produces a second new run that is indistinguishable from this one, and the shared concurrency group creates both records before either executes. Collect every new run instead and link only when exactly one exists; otherwise report the workflow page and say why. Linking the wrong run is worse than not linking one. The snapshot itself no longer swallows failure. An empty baseline makes every existing run look new, which guarantees a mislink, so it fails closed instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/beest-verification.yml | 60 +++++++++++++++++++++--- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/.github/workflows/beest-verification.yml b/.github/workflows/beest-verification.yml index a79f6b62c002..d5083f742e5f 100644 --- a/.github/workflows/beest-verification.yml +++ b/.github/workflows/beest-verification.yml @@ -155,6 +155,32 @@ jobs: *) echo "::error::unknown suite '${SUITE}'"; exit 1 ;; esac + # beest's scenario choices are architecture-specific and disjoint, so a value that + # is valid for one workflow is rejected by the other. Validate every target before + # dispatching any of them -- a half-dispatched suite still takes the global beest + # AWS lock. + scenarios_for() { + case "$1" in + agent-x86.yml) printf '%s\n' contd-eks-x86-1-36 contd-eks-x86-1-34 contd-eks-x86-1-35-fips all ;; + arm.yml) printf '%s\n' contd-eks-arm-1-36 contd-eks-arm-1-34 all ;; + esac + } + + if [ -n "${SCENARIOS}" ]; then + invalid=0 + for wf in "${workflows[@]}"; do + valid="$(scenarios_for "${wf}")" + if ! printf '%s\n' "${valid}" | grep -Fxq "${SCENARIOS}"; then + echo "::error::scenario '${SCENARIOS}' is not accepted by ${wf}; valid values: $(printf '%s' "${valid}" | tr '\n' ' ')" + invalid=1 + fi + done + if [ "${invalid}" -ne 0 ]; then + echo "::error::refusing to dispatch suite '${SUITE}' - use a scenario every selected workflow accepts ('all'), or leave it empty for each workflow's own default" + exit 1 + fi + fi + { echo "## Beest verification dispatched" echo @@ -180,26 +206,43 @@ jobs: # Snapshot the runs that already exist, because gh does not report the run it # queued and every beest AWS workflow shares one global lock -- so the newest - # run is routinely somebody else's, or an older queued one. - before="$(gh run list --repo StackVista/beest --workflow "${wf}" \ + # run is routinely somebody else's, or an older queued one. A failure here would + # leave the baseline empty and make every existing run look new, so it must not + # be swallowed. + if ! before="$(gh run list --repo StackVista/beest --workflow "${wf}" \ --branch "${BEEST_REF}" --limit 100 --json databaseId \ - --jq '[.[].databaseId | tostring] | join(",")' 2>/dev/null || true)" + --jq '[.[].databaseId | tostring] | join(",")')"; then + echo "::error::cannot list existing ${wf} runs; refusing to dispatch without a baseline" + exit 1 + fi echo "dispatching ${wf}" gh workflow run "${wf}" "${args[@]}" + # Link a run only when exactly one new one appeared: a concurrent dispatch of the + # same workflow and ref is indistinguishable from this one, and linking the wrong + # run sends someone to read unrelated results. url="" + note="" for _ in $(seq 1 30); do sleep 5 + candidates="" while read -r id candidate; do case ",${before}," in *",${id},"*) ;; - *) url="${candidate}"; break ;; + *) candidates="${candidates}${candidate}"$'\n' ;; esac done < <(gh run list --repo StackVista/beest --workflow "${wf}" \ --branch "${BEEST_REF}" --limit 100 --json databaseId,url \ --jq '.[] | "\(.databaseId) \(.url)"' 2>/dev/null || true) - if [ -n "${url}" ]; then + + count="$(printf '%s' "${candidates}" | grep -c . || true)" + if [ "${count}" -eq 1 ]; then + url="$(printf '%s' "${candidates}" | head -n 1)" + break + fi + if [ "${count}" -gt 1 ]; then + note="${count} new runs appeared, so this dispatch cannot be identified" break fi done @@ -208,7 +251,10 @@ jobs: echo "${wf} -> ${url}" echo "- \`${wf}\` -> ${url}" >> "${GITHUB_STEP_SUMMARY}" else - echo "::warning::dispatched ${wf} but no new run appeared within 150s" - echo "- \`${wf}\` -> https://github.com/StackVista/beest/actions/workflows/${wf} (new run not detected)" >> "${GITHUB_STEP_SUMMARY}" + if [ -z "${note}" ]; then + note="no new run appeared within 150s" + fi + echo "::warning::dispatched ${wf} but ${note}" + echo "- \`${wf}\` -> https://github.com/StackVista/beest/actions/workflows/${wf} (${note})" >> "${GITHUB_STEP_SUMMARY}" fi done