Review: skill-based prompts and an ensemble pipeline (benchmark candidate, do not merge)

.github/workflows/build-kai-ci.yml

@@ -42,6 +42,10 @@ on:
         type: choice
         default: both
         options: [both, agent, review, none]
+      candidate_kai_cli_ref:
+        description: 'Candidate only: build kai-cli from this branch or commit instead of its newest tag. Requires pin: none'
+        required: false
+        default: ''
 
 concurrency:
   group: build-kai-ci
@@ -93,12 +97,28 @@ jobs:
       # kai-engine's newest tag is v0.6.58 while its releases/latest still reads
       # v0.3.0, and kai-core publishes no releases at all — so that endpoint
       # would have silently pinned an engine 55 minor versions stale.
+      #
+      # A CANDIDATE is the one exception: kai-cli from a branch or commit, to
+      # trial a change on one org's reviews (org_review_config.review_image in
+      # kai-server) before it is released. It can never be pinned, so no
+      # production default is ever made from a branch tip, and the image still
+      # records the exact commit in its tag and labels.
       - name: Resolve refs
         id: resolve
         env:
           TOKEN: ${{ steps.token.outputs.token }}
+          CANDIDATE: ${{ inputs.candidate_kai_cli_ref }}
+          PIN: ${{ inputs.pin }}
         run: |
           set -euo pipefail
+          if [ -n "$CANDIDATE" ]; then
+            if [ "$PIN" != none ]; then
+              echo "a candidate build cannot be pinned: run it with pin: none"; exit 1
+            fi
+            if ! [[ "$CANDIDATE" =~ ^[A-Za-z0-9._/-]+$ ]]; then
+              echo "candidate ref must be a branch name or a commit"; exit 1
+            fi
+          fi
           # No pipe, deliberately. `git ls-remote | head -1` makes head close the
           # pipe while git is still writing, git takes SIGPIPE, and under
           # `set -o pipefail` the whole step dies with exit 141 — which is how
@@ -116,6 +136,11 @@ jobs:
             printf '%s' "${first##*refs/tags/}"
           }
           for m in kai-cli kai-tui kai-engine kai-core; do
+            if [ "$m" = kai-cli ] && [ -n "$CANDIDATE" ]; then
+              echo "$m: $CANDIDATE (candidate)"
+              echo "kai_cli_ref=$CANDIDATE" >> "$GITHUB_OUTPUT"
+              continue
+            fi
             # `if !` rather than a bare assignment: under `set -e` a failing
             # command substitution kills the step with no explanation, and the
             # message below never prints.

.github/workflows/review-regressions.yml

@@ -34,9 +34,15 @@ on:
       pin_pr:
         description: 'kai-server PR number to post the before/after on (PIN mode)'
         default: ''
+      record:
+        description: 'Append this run to the nightly dashboard (off for an ad-hoc comparison of two refs)'
+        type: boolean
+        default: true
 
 concurrency:
-  group: review-regressions-${{ inputs.pin_pr || 'nightly' }}
+  # An ad-hoc comparison (record off) runs on its own, so several can go at
+  # once; nightly and pin runs keep the shared group.
+  group: review-regressions-${{ inputs.pin_pr || (github.event_name == 'workflow_dispatch' && inputs.record == false && github.run_id) || 'nightly' }}
   cancel-in-progress: false
 
 permissions:
@@ -131,6 +137,7 @@ jobs:
 
       - name: Report
         id: report
+        if: ${{ inputs.record != false }}
         env:
           GH_TOKEN: ${{ github.token }}
           PIN_PR: ${{ inputs.pin_pr }}
@@ -183,6 +190,24 @@ jobs:
             gh issue comment "$ISSUE" --body-file "$RUNNER_TEMP/comment.md"
           fi
 
+      - name: Report (ad-hoc comparison, not recorded)
+        if: ${{ inputs.record == false }}
+        env:
+          KAI_REF: ${{ inputs.kai_ref || 'main' }}
+          BASELINE_REF: ${{ inputs.baseline_ref }}
+          SHA: ${{ steps.build.outputs.sha }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          set -euo pipefail
+          run_dir() { ls -d "$1"/runs/* 2>/dev/null | tail -1; }
+          args=(--run "$(run_dir "$RUNNER_TEMP/rr")" --cases "$REGRESSION_CASES" --ref "$KAI_REF" --sha "$SHA"
+                --run-url "$RUN_URL" --out-comment "$RUNNER_TEMP/comment.md")
+          if [ -n "$BASELINE_REF" ]; then
+            args+=(--baseline-run "$(run_dir "$RUNNER_TEMP/rr-base")" --baseline-ref "$BASELINE_REF")
+          fi
+          python3 scripts/review_regressions_report.py "${args[@]}"
+          cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY"
+
       - name: Mint App token for kai-server
         if: ${{ inputs.pin_pr != '' }}
         uses: actions/create-github-app-token@v3