From faaf81b843252f1375f006e4deaf6266d04b2b66 Mon Sep 17 00:00:00 2001 From: Chad Saunders Date: Thu, 12 Nov 2026 13:32:49 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20[fix]=20merge-gate:=20rerun=20a?= =?UTF-8?q?=20failed=20verify=20on=20the=20same=20tree=20before=20a=20reba?= =?UTF-8?q?se=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - new config key verify_retries (default 1, 0 = old behaviour) reruns a failed verify command on the same head and base, no rebuild - a pass on retry prints `flaky: ` on stdout and merges; failing every try still exits 2 - seen in a downstream run: a timing-flaky test failed verify once, the card bounced to Build and the rebuilt PR was identical - config-schema.json, run.md (exit 2 routing) and project-checks.md document it; test-merge-gate.sh scenarios 26-29 --- scripts/super-board-merge-gate.sh | 29 +++++++++++-- .../super-board/references/config-schema.json | 10 ++++- .../super-board/references/project-checks.md | 4 +- skills/super-board/references/run.md | 5 ++- tests/test-merge-gate.sh | 43 +++++++++++++++++++ 5 files changed, 83 insertions(+), 8 deletions(-) diff --git a/scripts/super-board-merge-gate.sh b/scripts/super-board-merge-gate.sh index 21c6dfeb..07a430d3 100755 --- a/scripts/super-board-merge-gate.sh +++ b/scripts/super-board-merge-gate.sh @@ -45,6 +45,13 @@ # Missing, empty, or whitespace-only commands halt with exit 9 before locking or # reading GitHub. Configure the owning project's checks; this gate invents none. # +# verify_retries: 1 (default) reruns a failed command on the same tree. +# +# A flaky test is not a stale branch. A timing test once failed verify at the +# gate; exit 2 sent the card to a rebase pass and the rebuilt PR was identical. +# The retry runs the same commit against the same base with no rebuild, so a +# pass on retry proves what a first-time pass does, minus the flake. 0 = no retry. +# # Usage: # super-board-merge-gate.sh --config --pr # [--expect-head ] [--subject ] [--body-file <md>] @@ -69,7 +76,10 @@ # # Exit codes, all meaningful to the caller: # 0 merged -# 2 verification failed — branch needs a rebase pass, NOT a Blocked card +# 2 verification failed on every try (1 + verify_retries) on the same tree — +# branch needs a rebase pass, NOT a Blocked card. A command that fails and +# then passes on a retry is a flake: stdout carries `flaky: <command>` and +# the merge goes ahead. # 3 PR is closed without a merge; no merge attempted # 4 could not take the lock within the timeout # 5 the base could not be merged in (real conflict) — needs a rebase pass @@ -160,6 +170,8 @@ if [ "${#VERIFY[@]}" -eq 0 ]; then echo "verification-readiness: verify_commands yielded no usable shell commands; repair the project config and restart the board." >&2 exit 9 fi +VERIFY_RETRIES=$(echo "$CONFIG_JSON" | jq -r '.verify_retries // 1') +case "$VERIFY_RETRIES" in ''|*[!0-9]*) echo "verify_retries must be a whole number (got: $VERIFY_RETRIES)" >&2; exit 64 ;; esac python3 "$GITHUB_READ" --check LOCK_DIR="${REPO_PATH}/.claude/super-board/inflight/merge.lock" @@ -280,11 +292,20 @@ fi for cmd in "${VERIFY[@]}"; do say "verify: ${cmd}" - if ! ( cd "$SCRATCH" && eval "$cmd" ) >"$SCRATCH/.verify.log" 2>&1; then + try=0 + until ( cd "$SCRATCH" && eval "$cmd" ) >"$SCRATCH/.verify.log" 2>&1; do say "FAILED: ${cmd}" tail -30 "$SCRATCH/.verify.log" >&2 - say "the branch is stale against ${BASE} — send it back to Build for a rebase pass" - exit 2 + if [ "$try" -ge "$VERIFY_RETRIES" ]; then + say "failed on every try — the branch is stale against ${BASE}; send it back to Build for a rebase pass" + exit 2 + fi + try=$(( try + 1 )) + say "retry ${try}/${VERIFY_RETRIES} on the same tree (${HEAD_SHA} + ${BASE})" + done + if [ "$try" -gt 0 ]; then + echo "flaky: ${cmd}" + say "flaky: ${cmd} failed, then passed on the same tree — not a rebase" fi done say "verified green against ${BASE} (${#VERIFY[@]} command(s))" diff --git a/skills/super-board/references/config-schema.json b/skills/super-board/references/config-schema.json index d9079d60..1034d5fd 100644 --- a/skills/super-board/references/config-schema.json +++ b/skills/super-board/references/config-schema.json @@ -259,8 +259,8 @@ // Shell commands proving the branch still BUILDS against the base branch as it // is right now. Run by super-board-merge-gate.sh from the root of a scratch // worktree that already has the current base merged in, inside the merge lock, - // immediately before the squash-merge. Any non-zero exit sends the card back - // to Build for a rebase pass; it never Blocks. + // immediately before the squash-merge. A non-zero exit on every try (see + // verify_retries) sends the card back to Build for a rebase pass; it never Blocks. // // Not optional in practice. GitHub's `mergeable: CLEAN` only answers "does the // text conflict" — it says nothing about whether the result compiles. On @@ -293,6 +293,12 @@ "cd server && npm ci --silent && npm run typecheck && npm test" ], + // How many times the merge gate reruns a failed verify command on the same + // scratch tree (same head, same base, no rebuild) before exit 2 sends the card + // to a rebase pass. Default 1; 0 = no retry. A pass on a retry prints + // `flaky: <command>` on stdout and merges: a flake is not a stale branch. + "verify_retries": 1, + // Optional throttle on how many cards a single wave dispatches. ABSENT OR 0 // MEANS UNLIMITED, which is the default and is usually what you want. // diff --git a/skills/super-board/references/project-checks.md b/skills/super-board/references/project-checks.md index d2bc8a2c..eadc58c9 100644 --- a/skills/super-board/references/project-checks.md +++ b/skills/super-board/references/project-checks.md @@ -19,7 +19,9 @@ contain: These are examples, not commands to invent in every app. Python/Rust/other projects use their own runners. The readiness gate checks that commands are configured; it cannot prove that a custom command tests the right behaviour. Failed checks -stop merging through the existing gate. +stop merging through the existing gate. A failed command is rerun on the same +tree `verify_retries` times (default 1, `0` = none) first; a pass on a retry +merges and prints `flaky: <command>` so the flaky check can be fixed. For import boundaries, enforce a decision the app already made, such as an internal server module being accessible only from server code. A browser app may diff --git a/skills/super-board/references/run.md b/skills/super-board/references/run.md index ac3b975b..f64bb013 100644 --- a/skills/super-board/references/run.md +++ b/skills/super-board/references/run.md @@ -477,7 +477,10 @@ on the base branch**. when the `.claude/hooks/cleanup-wt.py` hook is installed (local only; a cleanup failure never changes the exit). Route by exit code: 9 → verification-readiness: halt the run and preserve card, approval, claims, and worktree. Repair `config.verify_commands` with the project's real checks, then restart explicitly. Never bounce to Build, retry a rebase, or bypass the gate. 0 → merged; continue to step 4 - 2 → the branch no longer builds against the base → **rebase pass** (see below) + 2 → the branch no longer builds against the base (verify failed on every try, + including the gate's `verify_retries` reruns on the same tree) → **rebase pass** (see below). + A `flaky: <command>` stdout line means it failed and then passed on a retry: + the gate merged (exit 0); name the flaky command in the merged report, no rebase 3 → GitHub refused after a green verify (branch protection, required check) → Blocked with the §4 template; this one really is a human's 4 → another card holds the merge lock → leave the card in Review, next wave retries diff --git a/tests/test-merge-gate.sh b/tests/test-merge-gate.sh index 988fd036..68e9296c 100755 --- a/tests/test-merge-gate.sh +++ b/tests/test-merge-gate.sh @@ -545,4 +545,47 @@ RC=0; STUB_OID="$SHA" "$GATE" --config "$TMP/failing.json" --pr 1 --expect-head grep -q '^pr merge' "$GH_LOG" && fail "failed check must not merge" teardown +# 26 — a flaky verify is retried on the same tree, not sent to a rebase pass. A +# timing test failed verify once at the gate; exit 2 bounced the card to Build +# and the rebuilt PR was identical. `verify_retries` (default 1) reruns the +# failed command on the same scratch tree; a pass prints `flaky: <cmd>` and +# merges. Fails on its first run only, like a timing test under load: +FLAKY='n=$(cat COUNT 2>/dev/null || echo 0); echo $((n + 1)) > COUNT; [ "$n" -ge 1 ]' +head_setup +cfgcmd() { jq --arg c "$1" ".verify_commands = [\$c]${2:-}" "$TMP/c.json" > "$TMP/c2.json" && mv "$TMP/c2.json" "$TMP/c.json"; } +cfgcmd "${FLAKY//COUNT/$TMP/count}" +RC=0; OUT=$(STUB_OID="$SHA" gate --expect-head "$SHA" 2>"$TMP/err") || RC=$? +[ "$RC" -eq 0 ] || fail "a flake that passes on retry should merge (exit 0), got $RC: $(cat "$TMP/err")" +echo "$OUT" | grep -q '^flaky: ' || fail "a pass on retry must print flaky: <cmd> on stdout, got: $OUT" +grep -q 'retry 1/1 on the same tree' "$TMP/err" || fail "the retry must be on the same tree, got: $(cat "$TMP/err")" +[ "$(cat "$TMP/count")" = 2 ] || fail "the flaky command should run exactly twice, ran $(cat "$TMP/count")" +grep -q -- "--match-head-commit $SHA" "$GH_LOG" || fail "the flake should merge the reviewed head" +teardown + +# 27 — a real failure still exits 2 after the retry, and runs exactly 1 + retries times. +head_setup +cfgcmd "echo run >> $TMP/count; exit 1" ' | .verify_retries = 2' +RC=0; OUT=$(STUB_OID="$SHA" gate --expect-head "$SHA" 2>/dev/null) || RC=$? +[ "$RC" -eq 2 ] || fail "a command that fails every try must exit 2, got $RC" +[ "$(wc -l < "$TMP/count" | tr -d ' ')" = 3 ] || fail "verify_retries 2 should run the command 3 times, ran $(wc -l < "$TMP/count")" +echo "$OUT" | grep -q '^flaky:' && fail "a real failure is not a flake" +grep -q '^pr merge' "$GH_LOG" && fail "a real failure must not merge" +teardown + +# 28 — verify_retries: 0 keeps the old behaviour: one failure is exit 2. +head_setup +cfgcmd "${FLAKY//COUNT/$TMP/count}" ' | .verify_retries = 0' +RC=0; STUB_OID="$SHA" gate --expect-head "$SHA" >/dev/null 2>&1 || RC=$? +[ "$RC" -eq 2 ] || fail "verify_retries 0 must exit 2 on the first failure, got $RC" +[ "$(cat "$TMP/count")" = 1 ] || fail "verify_retries 0 must not retry, ran $(cat "$TMP/count")" +teardown + +# 29 — a verify_retries that is not a whole number is a usage error, before the lock. +head_setup +cfgset '.verify_retries = "twice"' +RC=0; STUB_OID="$SHA" gate --expect-head "$SHA" >/dev/null 2>&1 || RC=$? +[ "$RC" -eq 64 ] || fail "a bad verify_retries should exit 64, got $RC" +[ ! -d "$LOCK" ] || fail "a bad verify_retries must not take the merge lock" +teardown + echo "PASS: test-merge-gate.sh (merge, policy and approval scenarios)"