From 32d356f4267e7e907bcefe5eaefece143414990e Mon Sep 17 00:00:00 2001
From: wenjiefan <wenjiefan@microsoft.com>
Date: Thu, 3 Sep 2026 13:19:31 +0200
Subject: [PATCH] Allow evaluating an engine branch and only modified entries

Add optional engine-ref and modified-only inputs to pr-review-evaluation so an unmerged BCQuality article can be scored via a BC-ALAgents branch, without touching BC-Bench's BCQuality isolation. Both default to today's behaviour.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Copilot-Session: 7a9dff24-8942-4810-8359-18904eaa4da4
---
 .../install-agent-harnesses/action.yml        | 11 +++++++++-
 .github/workflows/pr-review-evaluation.yml    | 20 ++++++++++++++++---
 tests/test_review_workflows.py                |  4 ++--
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/.github/actions/install-agent-harnesses/action.yml b/.github/actions/install-agent-harnesses/action.yml
index 5d0ff334b..761eedb0a 100644
--- a/.github/actions/install-agent-harnesses/action.yml
+++ b/.github/actions/install-agent-harnesses/action.yml
@@ -1,6 +1,15 @@
 name: Install Agent Harnesses
 description: Install the agent harnesses used by the evaluation workflows
 
+inputs:
+  bc-alagents-ref:
+    description: >-
+      BC-ALAgents ref to evaluate. Leave empty to use the pinned baseline.
+      Override it to score an engine branch before it merges - for example a
+      branch whose bcquality.config.yaml points at an unmerged BCQuality PR.
+    required: false
+    default: ""
+
 outputs:
   bc-alagents-path:
     description: Path to the pinned BC-ALAgents checkout
@@ -21,7 +30,7 @@ runs:
       uses: actions/checkout@v5
       with:
         repository: microsoft/BC-ALAgents
-        ref: e9d9249eaa25be88388b60106b4cbf8cd7aa8a7d
+        ref: ${{ inputs.bc-alagents-ref || 'e9d9249eaa25be88388b60106b4cbf8cd7aa8a7d' }}
         path: .agent-harnesses/bc-alagents
         persist-credentials: false
 
diff --git a/.github/workflows/pr-review-evaluation.yml b/.github/workflows/pr-review-evaluation.yml
index 05cb8764f..ce7880c6c 100644
--- a/.github/workflows/pr-review-evaluation.yml
+++ b/.github/workflows/pr-review-evaluation.yml
@@ -41,9 +41,19 @@ on:
         required: false
         default: ""
         type: string
+      engine-ref:
+        description: "BC-ALAgents ref to evaluate (branch, tag, or SHA). Leave blank to use the pinned baseline engine."
+        required: false
+        default: ""
+        type: string
+      modified-only:
+        description: "Only evaluate dataset entries changed versus origin/main"
+        required: false
+        default: false
+        type: boolean
 
 concurrency:
-  group: pr-review-evaluation-${{ inputs.test-run && 'test' || 'full' }}
+  group: pr-review-evaluation-${{ inputs.test-run && 'test' || inputs.modified-only && 'modified' || 'full' }}
   cancel-in-progress: false
 
 env:
@@ -62,6 +72,7 @@ jobs:
   get-entries:
     uses: $/.github/workflows/get-entries.yml
     with:
+      modified-only: ${{ inputs.modified-only }}
       test-run: ${{ inputs.test-run }}
       category: code-review
 
@@ -111,6 +122,8 @@ jobs:
       - name: Install evaluation CLIs
         id: install-harnesses
         uses: $/.github/actions/install-agent-harnesses
+        with:
+          bc-alagents-ref: ${{ inputs.engine-ref }}
 
       - name: Run BC PR Review for entry ${{ matrix.entry }}
         timeout-minutes: 120
@@ -148,11 +161,12 @@ jobs:
       mock: ${{ inputs.test-run }}
       category: code-review
       git-ref: ${{ inputs.git-ref || github.ref_name }}
+      skip-leaderboard: ${{ inputs.modified-only }}
     secrets: inherit
 
   requeue:
     needs: [summarize-results, pin-commit]
-    if: ${{ !cancelled() && !failure() && !inputs.test-run }}
+    if: ${{ !cancelled() && !failure() && !inputs.test-run && !inputs.modified-only }}
     uses: $/.github/workflows/requeue-evaluation.yml
     permissions:
       contents: write
@@ -162,4 +176,4 @@ jobs:
       repeat: ${{ inputs.repeat }}
       existing-tag: ${{ needs.pin-commit.outputs.tag-name }}
       workflow-inputs: |
-        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}"}
+        {"model": "${{ inputs.model }}", "test-run": "${{ inputs.test-run }}", "git-ref": "${{ inputs.git-ref || github.ref_name }}", "engine-ref": "${{ inputs.engine-ref }}"}
diff --git a/tests/test_review_workflows.py b/tests/test_review_workflows.py
index 7548ccf02..a3f5d44ac 100644
--- a/tests/test_review_workflows.py
+++ b/tests/test_review_workflows.py
@@ -49,7 +49,7 @@ def test_pr_review_workflow_is_fixed_to_code_review() -> None:
     assert "mai-code-1-flash-picker" not in workflow
     assert '"gemini-3.7-flash"' in workflow
     assert "gemini-3.6-flash" not in workflow
-    for input_name in ("model:", "test-run:", "repeat:", "git-ref:"):
+    for input_name in ("model:", "test-run:", "repeat:", "git-ref:", "engine-ref:", "modified-only:"):
         assert input_name in workflow
 
 
@@ -63,5 +63,5 @@ def test_agent_harness_action_pins_and_exports_bc_alagents() -> None:
     action = (ACTIONS / "install-agent-harnesses" / "action.yml").read_text(encoding="utf-8")
 
     assert "repository: microsoft/BC-ALAgents" in action
-    assert "ref: e9d9249eaa25be88388b60106b4cbf8cd7aa8a7d" in action
+    assert "inputs.bc-alagents-ref || 'e9d9249eaa25be88388b60106b4cbf8cd7aa8a7d'" in action
     assert "bc-alagents-path:" in action