From 4d44c1b364f1d036bf3ff52d6a64bc5123fb19df Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Wed, 5 Aug 2026 15:02:35 -0500 Subject: [PATCH 1/6] ci: debug bonk reviewer hangs, move off deprecated kimi-k2.5 Every reviewer run since 2026-07-28 has hung at "Sending message to opencode..." and been killed at the 30 minute job timeout with no output. Last good run was 2026-07-27; every run since has failed. Three changes: - Swap cf-gateway/kimi-k2.5 for cf-gateway/nemotron-3-120b. @cf/moonshotai/kimi-k2.5 was deprecated on 2026-05-30 and now aliases to @cf/moonshotai/kimi-k2.6, which changed where reasoning content lands in the response (reasoning_content -> reasoning) and became Workers Paid only on 2026-07-28. @cf/nvidia/nemotron-3-120b-a12b keeps the 256k context, supports function calling, and is neither deprecated nor plan-gated. - Set OPENCODE_TIMEOUT to 25m. The harness defaults to 45m while both jobs cap at timeout-minutes: 30, so opencode's own timeout and retry handling could never fire and the runner killed it with no diagnostic. - Add a temporary Probe AI Gateway step that curls the gateway directly before opencode starts, logging status, latency, the model the gateway actually resolves to, and the raw stream shape for both models. Remove once the cause is confirmed. The kimi-k2.5 entry stays in opencode.json so the probe can compare the two side by side. --- .github/workflows/bonk.yml | 6 ++- .github/workflows/new-pr-review.yml | 60 ++++++++++++++++++++++++++++- .opencode/agents/reviewer.md | 2 +- opencode.json | 10 ++++- 4 files changed, 74 insertions(+), 4 deletions(-) diff --git a/.github/workflows/bonk.yml b/.github/workflows/bonk.yml index 0d74e43be0..ce9c33cd46 100644 --- a/.github/workflows/bonk.yml +++ b/.github/workflows/bonk.yml @@ -33,8 +33,12 @@ jobs: env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CF_GATEWAY_BASE_URL: https://gateway.ai.cloudflare.com/v1/${{ vars.CLOUDFLARE_ACCOUNT_ID }}/${{ vars.CLOUDFLARE_GATEWAY_ID }}/compat + # Must stay below the job's timeout-minutes. The harness defaults to + # 45m, so on a 30m job opencode's own timeout and retry handling never + # fires and the runner kills it with no diagnostic. + OPENCODE_TIMEOUT: 25m with: - model: 'cf-gateway/kimi-k2.5' + model: 'cf-gateway/nemotron-3-120b' mentions: '/bonk,@ask-bonk' forks: 'false' permissions: write diff --git a/.github/workflows/new-pr-review.yml b/.github/workflows/new-pr-review.yml index 0cfa67fec6..baf9bc7315 100644 --- a/.github/workflows/new-pr-review.yml +++ b/.github/workflows/new-pr-review.yml @@ -34,13 +34,71 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" + # TEMPORARY (debug/bonk-reviewer-hang): every reviewer run since + # 2026-07-28 has hung at "Sending message to opencode..." and been killed + # at the 30 minute job timeout with zero output. This probe hits the + # gateway directly so the job log shows the status code, latency, the + # model the gateway actually resolves to, and the raw shape of the + # stream. Delete this step once the cause is known. + - name: Probe AI Gateway + continue-on-error: true + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CF_GATEWAY_BASE_URL: https://gateway.ai.cloudflare.com/v1/${{ vars.CLOUDFLARE_ACCOUNT_ID }}/${{ vars.CLOUDFLARE_GATEWAY_ID }}/compat + run: | + # Diagnostics must never fail the job, and a hung or erroring probe is + # itself the signal we want, so drop the default -e / pipefail. + set +e +o pipefail + + probe() { + local model="$1" + local stream="$2" + local prompt="$3" + local body hdr stats + echo "::group::model=${model} stream=${stream}" + body="$(mktemp)" + hdr="$(mktemp)" + stats="$( + curl -sS -X POST "${CF_GATEWAY_BASE_URL}/chat/completions" \ + -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \ + -H 'Content-Type: application/json' \ + --max-time 120 \ + -D "${hdr}" -o "${body}" \ + -w 'http=%{http_code} ttfb=%{time_starttransfer}s total=%{time_total}s bytes=%{size_download}' \ + -d "{\"model\":\"${model}\",\"stream\":${stream},\"max_tokens\":256,\"messages\":[{\"role\":\"user\",\"content\":\"${prompt}\"}]}" + )" || echo "curl failed with exit code $?" + echo "${stats}" + echo "-- response headers --" + grep -iv '^set-cookie' "${hdr}" | head -n 40 || true + echo "-- body (first 3000 bytes) --" + head -c 3000 "${body}" + echo + echo "::endgroup::" + } + + # Non-streaming. The "model" field in the response reveals whether the + # gateway silently aliases deprecated kimi-k2.5 to kimi-k2.6. + probe 'workers-ai/@cf/moonshotai/kimi-k2.5' false 'Reply with the single word: pong' + probe 'workers-ai/@cf/nvidia/nemotron-3-120b-a12b' false 'Reply with the single word: pong' + + # Streaming with a prompt that forces reasoning. Shows whether tokens + # arrive as content, reasoning, or reasoning_content. If they land in + # a field the openai-compatible provider ignores, opencode would print + # nothing while the model runs, which matches the observed hang. + probe 'workers-ai/@cf/moonshotai/kimi-k2.5' true 'What is 17 times 23? Think step by step.' + probe 'workers-ai/@cf/nvidia/nemotron-3-120b-a12b' true 'What is 17 times 23? Think step by step.' + - name: Run Bonk uses: ask-bonk/ask-bonk/github@main env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CF_GATEWAY_BASE_URL: https://gateway.ai.cloudflare.com/v1/${{ vars.CLOUDFLARE_ACCOUNT_ID }}/${{ vars.CLOUDFLARE_GATEWAY_ID }}/compat + # Must stay below the job's timeout-minutes. The harness defaults to + # 45m, so on a 30m job opencode's own timeout and retry handling never + # fires and the runner kills it with no diagnostic. + OPENCODE_TIMEOUT: 25m with: - model: 'cf-gateway/kimi-k2.5' + model: 'cf-gateway/nemotron-3-120b' forks: 'false' permissions: write opencode_version: '1.2.27' diff --git a/.opencode/agents/reviewer.md b/.opencode/agents/reviewer.md index 26ea3950d3..37d7ead973 100644 --- a/.opencode/agents/reviewer.md +++ b/.opencode/agents/reviewer.md @@ -1,7 +1,7 @@ --- description: Read-only code reviewer for pull requests mode: primary -model: cf-gateway/kimi-k2.5 +model: cf-gateway/nemotron-3-120b temperature: 0.1 permission: edit: deny diff --git a/opencode.json b/opencode.json index 74f2b5897e..1d6b08708c 100644 --- a/opencode.json +++ b/opencode.json @@ -9,9 +9,17 @@ "apiKey": "{env:CLOUDFLARE_API_TOKEN}" }, "models": { + "nemotron-3-120b": { + "id": "workers-ai/@cf/nvidia/nemotron-3-120b-a12b", + "name": "Nemotron 3 120B (Workers AI via Gateway)", + "limit": { + "context": 256000, + "output": 32000 + } + }, "kimi-k2.5": { "id": "workers-ai/@cf/moonshotai/kimi-k2.5", - "name": "Kimi K2.5 (Workers AI via Gateway)", + "name": "Kimi K2.5 (Workers AI via Gateway, deprecated)", "limit": { "context": 256000, "output": 64000 From b33675c050167042e4b189c70d53a5ac1225f0b8 Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Wed, 5 Aug 2026 15:15:55 -0500 Subject: [PATCH 2/6] ci: surface opencode logs, bump pinned opencode to 1.18.13 Replaces the throwaway gateway probe with opencode's own logging. The reviewer hang was diagnosable in seconds once we saw the response: the gateway returned 429 "inference request per min rate reached" for the deprecated @cf/moonshotai/kimi-k2.5. opencode retried it silently and printed nothing, so the job just sat there until the runner killed it. The error existed the whole time, it was only ever written to opencode's log file inside the runner. --print-logs and --log-level are plain sugar over OPENCODE_PRINT_LOGS and OPENCODE_LOG_LEVEL (packages/opencode/src/index.ts:67-68), and the harness spawns opencode with the ambient environment, so setting the two variables streams the logs to stderr and into the job log. No flags to plumb through and nothing to dump after the fact. Also bumps the pinned opencode from 1.2.27, which is roughly 11,700 versions behind, to the current 1.18.13. Drops the kimi-k2.5 entry from opencode.json now that nothing references it. --- .github/workflows/bonk.yml | 9 ++++- .github/workflows/new-pr-review.yml | 63 ++++------------------------- opencode.json | 8 ---- 3 files changed, 16 insertions(+), 64 deletions(-) diff --git a/.github/workflows/bonk.yml b/.github/workflows/bonk.yml index ce9c33cd46..dbaa79f6f1 100644 --- a/.github/workflows/bonk.yml +++ b/.github/workflows/bonk.yml @@ -37,11 +37,18 @@ jobs: # 45m, so on a 30m job opencode's own timeout and retry handling never # fires and the runner kills it with no diagnostic. OPENCODE_TIMEOUT: 25m + # The equivalent of --print-logs and --log-level, which we cannot pass + # as flags because the harness spawns `opencode github run` directly. + # Without these, opencode only writes to its log file inside the + # runner, so a provider error looks like a silent stall: a deprecated + # model returning 429 cost us a week of timeouts with an empty log. + OPENCODE_PRINT_LOGS: '1' + OPENCODE_LOG_LEVEL: DEBUG with: model: 'cf-gateway/nemotron-3-120b' mentions: '/bonk,@ask-bonk' forks: 'false' permissions: write - opencode_version: '1.2.27' + opencode_version: '1.18.13' # token_permissions defaults to WRITE so bonk can push commits # when asked via /bonk. diff --git a/.github/workflows/new-pr-review.yml b/.github/workflows/new-pr-review.yml index baf9bc7315..3eb2eb64ef 100644 --- a/.github/workflows/new-pr-review.yml +++ b/.github/workflows/new-pr-review.yml @@ -34,60 +34,6 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" - # TEMPORARY (debug/bonk-reviewer-hang): every reviewer run since - # 2026-07-28 has hung at "Sending message to opencode..." and been killed - # at the 30 minute job timeout with zero output. This probe hits the - # gateway directly so the job log shows the status code, latency, the - # model the gateway actually resolves to, and the raw shape of the - # stream. Delete this step once the cause is known. - - name: Probe AI Gateway - continue-on-error: true - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CF_GATEWAY_BASE_URL: https://gateway.ai.cloudflare.com/v1/${{ vars.CLOUDFLARE_ACCOUNT_ID }}/${{ vars.CLOUDFLARE_GATEWAY_ID }}/compat - run: | - # Diagnostics must never fail the job, and a hung or erroring probe is - # itself the signal we want, so drop the default -e / pipefail. - set +e +o pipefail - - probe() { - local model="$1" - local stream="$2" - local prompt="$3" - local body hdr stats - echo "::group::model=${model} stream=${stream}" - body="$(mktemp)" - hdr="$(mktemp)" - stats="$( - curl -sS -X POST "${CF_GATEWAY_BASE_URL}/chat/completions" \ - -H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \ - -H 'Content-Type: application/json' \ - --max-time 120 \ - -D "${hdr}" -o "${body}" \ - -w 'http=%{http_code} ttfb=%{time_starttransfer}s total=%{time_total}s bytes=%{size_download}' \ - -d "{\"model\":\"${model}\",\"stream\":${stream},\"max_tokens\":256,\"messages\":[{\"role\":\"user\",\"content\":\"${prompt}\"}]}" - )" || echo "curl failed with exit code $?" - echo "${stats}" - echo "-- response headers --" - grep -iv '^set-cookie' "${hdr}" | head -n 40 || true - echo "-- body (first 3000 bytes) --" - head -c 3000 "${body}" - echo - echo "::endgroup::" - } - - # Non-streaming. The "model" field in the response reveals whether the - # gateway silently aliases deprecated kimi-k2.5 to kimi-k2.6. - probe 'workers-ai/@cf/moonshotai/kimi-k2.5' false 'Reply with the single word: pong' - probe 'workers-ai/@cf/nvidia/nemotron-3-120b-a12b' false 'Reply with the single word: pong' - - # Streaming with a prompt that forces reasoning. Shows whether tokens - # arrive as content, reasoning, or reasoning_content. If they land in - # a field the openai-compatible provider ignores, opencode would print - # nothing while the model runs, which matches the observed hang. - probe 'workers-ai/@cf/moonshotai/kimi-k2.5' true 'What is 17 times 23? Think step by step.' - probe 'workers-ai/@cf/nvidia/nemotron-3-120b-a12b' true 'What is 17 times 23? Think step by step.' - - name: Run Bonk uses: ask-bonk/ask-bonk/github@main env: @@ -97,11 +43,18 @@ jobs: # 45m, so on a 30m job opencode's own timeout and retry handling never # fires and the runner kills it with no diagnostic. OPENCODE_TIMEOUT: 25m + # The equivalent of --print-logs and --log-level, which we cannot pass + # as flags because the harness spawns `opencode github run` directly. + # Without these, opencode only writes to its log file inside the + # runner, so a provider error looks like a silent stall: a deprecated + # model returning 429 cost us a week of timeouts with an empty log. + OPENCODE_PRINT_LOGS: '1' + OPENCODE_LOG_LEVEL: DEBUG with: model: 'cf-gateway/nemotron-3-120b' forks: 'false' permissions: write - opencode_version: '1.2.27' + opencode_version: '1.18.13' # The auto-reviewer must never push to PR branches. NO_PUSH # enforces that at the token level. token_permissions: 'NO_PUSH' diff --git a/opencode.json b/opencode.json index 1d6b08708c..74d8f349eb 100644 --- a/opencode.json +++ b/opencode.json @@ -16,14 +16,6 @@ "context": 256000, "output": 32000 } - }, - "kimi-k2.5": { - "id": "workers-ai/@cf/moonshotai/kimi-k2.5", - "name": "Kimi K2.5 (Workers AI via Gateway, deprecated)", - "limit": { - "context": 256000, - "output": 64000 - } } } } From f80f6d2c724d043effc541c89ecca72d3ec04b56 Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Wed, 5 Aug 2026 15:17:44 -0500 Subject: [PATCH 3/6] ci: log opencode at INFO rather than DEBUG DEBUG is more noise than the reviewer job needs. INFO still carries provider errors, which is the whole point of turning the logs on. --- .github/workflows/bonk.yml | 2 +- .github/workflows/new-pr-review.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bonk.yml b/.github/workflows/bonk.yml index dbaa79f6f1..6060ed66ed 100644 --- a/.github/workflows/bonk.yml +++ b/.github/workflows/bonk.yml @@ -43,7 +43,7 @@ jobs: # runner, so a provider error looks like a silent stall: a deprecated # model returning 429 cost us a week of timeouts with an empty log. OPENCODE_PRINT_LOGS: '1' - OPENCODE_LOG_LEVEL: DEBUG + OPENCODE_LOG_LEVEL: INFO with: model: 'cf-gateway/nemotron-3-120b' mentions: '/bonk,@ask-bonk' diff --git a/.github/workflows/new-pr-review.yml b/.github/workflows/new-pr-review.yml index 3eb2eb64ef..97d53e253c 100644 --- a/.github/workflows/new-pr-review.yml +++ b/.github/workflows/new-pr-review.yml @@ -49,7 +49,7 @@ jobs: # runner, so a provider error looks like a silent stall: a deprecated # model returning 429 cost us a week of timeouts with an empty log. OPENCODE_PRINT_LOGS: '1' - OPENCODE_LOG_LEVEL: DEBUG + OPENCODE_LOG_LEVEL: INFO with: model: 'cf-gateway/nemotron-3-120b' forks: 'false' From dc1001fcd20d43cede775c7053b5cac73fa39f16 Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Wed, 5 Aug 2026 15:25:27 -0500 Subject: [PATCH 4/6] Adjust OPENCODE_TIMEOUT in bonk.yml Updated OPENCODE_TIMEOUT to 25m for better retry handling. Signed-off-by: Anish Sahoo --- .github/workflows/bonk.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/bonk.yml b/.github/workflows/bonk.yml index 6060ed66ed..5e70685b48 100644 --- a/.github/workflows/bonk.yml +++ b/.github/workflows/bonk.yml @@ -33,15 +33,8 @@ jobs: env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CF_GATEWAY_BASE_URL: https://gateway.ai.cloudflare.com/v1/${{ vars.CLOUDFLARE_ACCOUNT_ID }}/${{ vars.CLOUDFLARE_GATEWAY_ID }}/compat - # Must stay below the job's timeout-minutes. The harness defaults to - # 45m, so on a 30m job opencode's own timeout and retry handling never - # fires and the runner kills it with no diagnostic. + # Must stay below the job's timeout-minutes. The harness defaults to 45m, so 25m gives opencode enough time to retry OPENCODE_TIMEOUT: 25m - # The equivalent of --print-logs and --log-level, which we cannot pass - # as flags because the harness spawns `opencode github run` directly. - # Without these, opencode only writes to its log file inside the - # runner, so a provider error looks like a silent stall: a deprecated - # model returning 429 cost us a week of timeouts with an empty log. OPENCODE_PRINT_LOGS: '1' OPENCODE_LOG_LEVEL: INFO with: From 990990ff89f9ed3ec2b4f1c6817e5cafbc0ea067 Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Wed, 5 Aug 2026 15:30:33 -0500 Subject: [PATCH 5/6] Adjust OPENCODE_TIMEOUT in new-pr-review.yml Updated OPENCODE_TIMEOUT to provide more retry time. Signed-off-by: Anish Sahoo --- .github/workflows/new-pr-review.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/new-pr-review.yml b/.github/workflows/new-pr-review.yml index 97d53e253c..7629460545 100644 --- a/.github/workflows/new-pr-review.yml +++ b/.github/workflows/new-pr-review.yml @@ -39,15 +39,8 @@ jobs: env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CF_GATEWAY_BASE_URL: https://gateway.ai.cloudflare.com/v1/${{ vars.CLOUDFLARE_ACCOUNT_ID }}/${{ vars.CLOUDFLARE_GATEWAY_ID }}/compat - # Must stay below the job's timeout-minutes. The harness defaults to - # 45m, so on a 30m job opencode's own timeout and retry handling never - # fires and the runner kills it with no diagnostic. + # Must stay below the job's timeout-minutes. The harness defaults to 45m, so 25m gives opencode enough time to retry OPENCODE_TIMEOUT: 25m - # The equivalent of --print-logs and --log-level, which we cannot pass - # as flags because the harness spawns `opencode github run` directly. - # Without these, opencode only writes to its log file inside the - # runner, so a provider error looks like a silent stall: a deprecated - # model returning 429 cost us a week of timeouts with an empty log. OPENCODE_PRINT_LOGS: '1' OPENCODE_LOG_LEVEL: INFO with: From 173804ff8ec58aa3dd3bc8330f5fec2b6b80dfe4 Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Wed, 5 Aug 2026 15:52:11 -0500 Subject: [PATCH 6/6] ci: use Kimi K2.7 Code for Bonk --- .github/workflows/bonk.yml | 2 +- .github/workflows/new-pr-review.yml | 2 +- .opencode/agents/reviewer.md | 2 +- opencode.json | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bonk.yml b/.github/workflows/bonk.yml index 5e70685b48..4d4b40cdd2 100644 --- a/.github/workflows/bonk.yml +++ b/.github/workflows/bonk.yml @@ -38,7 +38,7 @@ jobs: OPENCODE_PRINT_LOGS: '1' OPENCODE_LOG_LEVEL: INFO with: - model: 'cf-gateway/nemotron-3-120b' + model: 'cf-gateway/kimi-k2.7-code' mentions: '/bonk,@ask-bonk' forks: 'false' permissions: write diff --git a/.github/workflows/new-pr-review.yml b/.github/workflows/new-pr-review.yml index 7629460545..be795656fe 100644 --- a/.github/workflows/new-pr-review.yml +++ b/.github/workflows/new-pr-review.yml @@ -44,7 +44,7 @@ jobs: OPENCODE_PRINT_LOGS: '1' OPENCODE_LOG_LEVEL: INFO with: - model: 'cf-gateway/nemotron-3-120b' + model: 'cf-gateway/kimi-k2.7-code' forks: 'false' permissions: write opencode_version: '1.18.13' diff --git a/.opencode/agents/reviewer.md b/.opencode/agents/reviewer.md index 37d7ead973..073fd06d3b 100644 --- a/.opencode/agents/reviewer.md +++ b/.opencode/agents/reviewer.md @@ -1,7 +1,7 @@ --- description: Read-only code reviewer for pull requests mode: primary -model: cf-gateway/nemotron-3-120b +model: cf-gateway/kimi-k2.7-code temperature: 0.1 permission: edit: deny diff --git a/opencode.json b/opencode.json index 74d8f349eb..c78f37eb50 100644 --- a/opencode.json +++ b/opencode.json @@ -9,12 +9,12 @@ "apiKey": "{env:CLOUDFLARE_API_TOKEN}" }, "models": { - "nemotron-3-120b": { - "id": "workers-ai/@cf/nvidia/nemotron-3-120b-a12b", - "name": "Nemotron 3 120B (Workers AI via Gateway)", + "kimi-k2.7-code": { + "id": "workers-ai/@cf/moonshotai/kimi-k2.7-code", + "name": "Kimi K2.7 Code (Workers AI via Gateway)", "limit": { - "context": 256000, - "output": 32000 + "context": 262144, + "output": 64000 } } }