From c67df2e67913d9bba93b539a3fe31ac903a99cfc Mon Sep 17 00:00:00 2001 From: Tiberiu Socaci Date: Fri, 6 Nov 2026 22:09:52 +0200 Subject: [PATCH 1/2] fix: format background continuation errors for chat Signed-off-by: Tiberiu Socaci --- CHANGELOG.md | 3 + FEATURES.md | 4 + TEST-PLAN.md | 12 +++ src/gateway/background.js | 6 +- test/background-continuation-errors.test.js | 97 +++++++++++++++++++++ 5 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 test/background-continuation-errors.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index e6ce1ef..0bc6707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## 0.6.5 — candidate, pending approval +- Show readable, redacted provider errors when a background job's automatic follow-up fails, + preserving diagnostic audit detail and protection against replaying interrupted work. + Prepared 2026-11-06 from integrated beta development. This is not a publication date. Requires Node.js 24.0.0 or newer and runtime image spec 1.6.3. See [the candidate record](docs/releases/0.6.5.md) for scope, excluded unfinished work and diff --git a/FEATURES.md b/FEATURES.md index 398c41d..b2125b4 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -1,5 +1,9 @@ # ChannelGate — Features +- **Readable background continuation errors.** Failed follow-ups show a bounded provider sentence + instead of a nested JSON response. Redacted audit diagnostics retain their detail; structured + engine failure facts and the guard against replaying interrupted continuations remain unchanged. + - **Continue after variable server approval in Slack.** Approving a hidden variable’s server automatically queues one continuation in the original thread as the original requester. It waits behind active work, honors queued Stop and current access, records usage and deduplicates repeated clicks. Queued wakeups survive restart; interrupted continuations are reported without replaying unknown actions. Denial and stale approvals do not continue. The documentation handbook includes its branded favicon in standalone builds and website exports. diff --git a/TEST-PLAN.md b/TEST-PLAN.md index 4235e8d..83e72a3 100644 --- a/TEST-PLAN.md +++ b/TEST-PLAN.md @@ -1,5 +1,17 @@ # ChannelGate — Test Plan +## Readable background continuation errors (2026-11-06) + +`node --test test/background-continuation-errors.test.js test/process-outcome.test.js` +executes BackgroundJobs through runMessage and the real Claude/Codex parsers, with a spawned +fixture CLI emitting each engine's nested provider-error event. Assert readable notices without +JSON, secret redaction in notices and audit, preserved provider facts and diagnostic codes, +released runtime leases, and no second spawn when delivery resumes after the failed continuation. +The runtime backend and Slack delivery boundary are fixtures, not live provider/client acceptance. +Private REL-RUNTIME-07 remains the per-engine acceptance gate: reproduce the nested provider +failure in its isolated acceptance setup, inspect the actual continuation notice and durable +recovery, and keep deployed/native observation distinct from these automated results. + ## Automatic continuation after variable server approval Automated: `node --test test/secret-host-continuation.test.js test/egress-secret-approval.test.js test/durable-approvals.test.js test/approval-links.test.js`. diff --git a/src/gateway/background.js b/src/gateway/background.js index 335a04c..a616623 100644 --- a/src/gateway/background.js +++ b/src/gateway/background.js @@ -15,7 +15,7 @@ import { buildShellEnv, serviceSecretValues } from "../engines/child-env.js"; import { resolveRuntime } from "../runtimes/resolve.js"; import { newRunId, runtimeSupports } from "../runtimes/contract.js"; import { killGroup, killTree, processStartTime } from "../util/proc.js"; -import { describeProcessOutcome } from "../util/process-outcome.js"; +import { describeProcessOutcome, plainFailureText } from "../util/process-outcome.js"; import { appendTail } from "../util/tail.js"; import { effectiveWorkDir } from "./folders.js"; import { gatewayRoot } from "../config/paths.js"; @@ -907,8 +907,10 @@ export class BackgroundJobs { // Boot recovery redelivers it (up to MAX_DELIVERY_ATTEMPTS). const failure = redactSecretValues(err.message, outputSecrets); await logEvent("bg_continue_error", { id: rec.id, error: failure }); + // Keep diagnostic detail in the audit, but show a bounded sentence at the chat boundary. + const noticeFailure = redactSecretValues(plainFailureText(err.message) || "The continuation failed.", outputSecrets); try { - await postNotice(client, { conversationId: rec.channelId, threadKey: rec.threadKey, text: `⚠️ ${what} *${rec.label}* ${status}, but I couldn't continue: ${failure}` }); + await postNotice(client, { conversationId: rec.channelId, threadKey: rec.threadKey, text: `⚠️ ${what} *${rec.label}* ${status}, but I couldn't continue: ${noticeFailure}` }); } catch { /* ignore */ } diff --git a/test/background-continuation-errors.test.js b/test/background-continuation-errors.test.js new file mode 100644 index 0000000..04dbafa --- /dev/null +++ b/test/background-continuation-errors.test.js @@ -0,0 +1,97 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import path from "node:path"; +import { writeFileSync } from "node:fs"; +import { ensureTestEnv } from "./helpers.js"; + +const scratch = ensureTestEnv(); +process.env.SESSION_KEEPALIVE = "0"; +process.env.PATH = path.join(import.meta.dirname, "fixtures") + path.delimiter + process.env.PATH; + +const { useFakeRuntime, createFakeRuntimeBackend } = await import("./runtime-fake.js"); +const backend = createFakeRuntimeBackend(); +const spawn = backend.spawn.bind(backend); +const cli = path.join(scratch, "provider-error.mjs"); +const canary = "synthetic-continuation-secret"; +writeFileSync(cli, ` +const message = JSON.stringify({ error: { + message: "The selected model is unavailable. ${canary}", code: "fixture_model_unavailable" +} }); +const event = process.argv[2] === "codex" + ? { type: "turn.failed", error: { message } } + : { type: "assistant", error: "invalid_request_error", message: { + role: "assistant", content: [{ type: "text", text: message }] + } }; +console.log(JSON.stringify(event)); +process.exitCode = 1; +`); +let engine; +backend.spawn = (target, spec) => spawn(target, { + ...spec, cmd: process.execPath, args: [cli, engine], +}); +await useFakeRuntime(backend); + +const { setUser, upsertChannelEntry, saveChannelMeta } = await import("../src/config/store.js"); +const { saveSettings } = await import("../src/config/settings.js"); +const { runMessage } = await import("../src/gateway/run.js"); +const { BackgroundJobs } = await import("../src/gateway/background.js"); +const { readEvents } = await import("../src/util/logger.js"); +const { getDb } = await import("../src/db/index.js"); + +for (const selected of ["claude", "codex"]) { + test(`${selected} CLI continuation failures stay readable, redacted and unreplayed`, async () => { + engine = selected; + saveSettings({ engine, engineFallback: false, engineEnabled: { claude: true, codex: true }, composioMode: "personal" }); + await setUser("U_CONTINUATION", { name: "Fixture admin", isAdmin: true, approved: true }); + const channel = `D_CONTINUATION_${engine.toUpperCase()}`; + const entry = await upsertChannelEntry(channel, { name: `continuation-${engine}`, type: "im", isDM: true }); + await saveChannelMeta(entry.slug, { channelId: channel, type: "im", isDM: true, template: "custom", engine, cleanMode: true, allowNetwork: false }); + const posts = [], errors = []; + const client = { chat: { postMessage: async payload => { + posts.push(payload); + return { ok: true, ts: "1900000000.000001" }; + } } }; + const jobs = new BackgroundJobs({ + slack: { snapshot: () => ({ connected: true }), getClient: () => client }, + runner: async args => { + try { return await runMessage({ ...args, preferCold: true }); } + catch (error) { errors.push(error); throw error; } + }, + }); + const rec = { + id: `continuation-${engine}`, kind: "shell", channelId: channel, slug: entry.slug, + authorId: "U_CONTINUATION", threadKey: "1900000000.000002", label: "fixture job", + command: "true", startedAt: Date.now() - 1000, pid: null, maxMs: 60000, + logFile: "", tail: "harmless output", timedOut: false, secretValues: [canary], + }; + jobs.jobs.set(rec.id, rec); + try { + const before = backend.calls.spawn.length; + // Real engine parser and runMessage; only runtime execution and transport are fixtures. + await jobs._finish(rec, { code: 0 }); + assert.equal(backend.calls.spawn.length, before + 1); + assert.equal(errors.length, 1); + assert.equal(errors[0].details.engine, engine); + assert.equal(errors[0].details.providerError, true); + assert.ok(errors[0].details.providerCode); + assert.equal(errors[0].details.exitCode, 1); + const notice = posts.find(post => post.text.includes("couldn't continue"))?.text; + assert.match(notice, /The selected model is unavailable\. \[REDACTED\]/); + assert.doesNotMatch(notice, /\{\s*"error"|fixture_model_unavailable/); + assert.ok(!notice.includes(canary)); + const audit = readEvents({ limit: 200 }).find(row => row.event === "bg_continue_error" && row.id === rec.id); + assert.ok(audit); + assert.ok(!audit.error.includes(canary)); + if (engine === "claude") assert.match(audit.error, /fixture_model_unavailable/, "redacted diagnostic detail is preserved"); + await jobs._deliver(rec); + assert.equal(backend.calls.spawn.length, before + 1, "unknown continuation effects are never replayed"); + assert.ok(posts.some(post => /continuation was interrupted/.test(post.text))); + assert.equal(jobs.jobs.has(rec.id), false); + assert.equal(getDb().prepare("SELECT id FROM bg_jobs WHERE id = ?").get(rec.id), undefined); + assert.ok(backend.calls.leases.every(lease => lease.released)); + } finally { + jobs.jobs.clear(); + getDb().prepare("DELETE FROM bg_jobs WHERE id = ?").run(rec.id); + } + }); +} From b1e829960698ae13aaa0d9d61630d7eeac8cd2f0 Mon Sep 17 00:00:00 2001 From: Tiberiu Socaci Date: Fri, 6 Nov 2026 22:12:36 +0200 Subject: [PATCH 2/2] fix: redact decoded continuation errors before bounding Signed-off-by: Tiberiu Socaci --- TEST-PLAN.md | 3 +- src/gateway/background.js | 7 +- test/background-continuation-errors.test.js | 125 +++++++++++--------- 3 files changed, 74 insertions(+), 61 deletions(-) diff --git a/TEST-PLAN.md b/TEST-PLAN.md index 83e72a3..9fbf5ed 100644 --- a/TEST-PLAN.md +++ b/TEST-PLAN.md @@ -5,7 +5,8 @@ `node --test test/background-continuation-errors.test.js test/process-outcome.test.js` executes BackgroundJobs through runMessage and the real Claude/Codex parsers, with a spawned fixture CLI emitting each engine's nested provider-error event. Assert readable notices without -JSON, secret redaction in notices and audit, preserved provider facts and diagnostic codes, +JSON, secret redaction in notices and audit, no partial secret at the 400-character boundary, +redaction after JSON escape decoding, preserved provider facts and diagnostic codes, released runtime leases, and no second spawn when delivery resumes after the failed continuation. The runtime backend and Slack delivery boundary are fixtures, not live provider/client acceptance. Private REL-RUNTIME-07 remains the per-engine acceptance gate: reproduce the nested provider diff --git a/src/gateway/background.js b/src/gateway/background.js index a616623..a606d4d 100644 --- a/src/gateway/background.js +++ b/src/gateway/background.js @@ -907,8 +907,11 @@ export class BackgroundJobs { // Boot recovery redelivers it (up to MAX_DELIVERY_ATTEMPTS). const failure = redactSecretValues(err.message, outputSecrets); await logEvent("bg_continue_error", { id: rec.id, error: failure }); - // Keep diagnostic detail in the audit, but show a bounded sentence at the chat boundary. - const noticeFailure = redactSecretValues(plainFailureText(err.message) || "The continuation failed.", outputSecrets); + // Decode before the final redaction, and redact before bounding: truncation must never + // leave a partial secret that the full-value redactor can no longer recognize. + const noticeFailure = redactSecretValues( + plainFailureText(failure, Number.MAX_SAFE_INTEGER) || "The continuation failed.", outputSecrets, + ).slice(0, 400); try { await postNotice(client, { conversationId: rec.channelId, threadKey: rec.threadKey, text: `⚠️ ${what} *${rec.label}* ${status}, but I couldn't continue: ${noticeFailure}` }); } catch { diff --git a/test/background-continuation-errors.test.js b/test/background-continuation-errors.test.js index 04dbafa..cf357ed 100644 --- a/test/background-continuation-errors.test.js +++ b/test/background-continuation-errors.test.js @@ -14,9 +14,14 @@ const spawn = backend.spawn.bind(backend); const cli = path.join(scratch, "provider-error.mjs"); const canary = "synthetic-continuation-secret"; writeFileSync(cli, ` -const message = JSON.stringify({ error: { - message: "The selected model is unavailable. ${canary}", code: "fixture_model_unavailable" +const padding = process.argv[2] === "claude" ? 390 - "Claude provider rejected the request: ".length : 390; +const body = process.argv[3] === "long" + ? "x".repeat(padding) + "${canary}" + : "The selected model is unavailable. ${canary}"; +let message = JSON.stringify({ error: { + message: body, code: "fixture_model_unavailable" } }); +if (process.argv[3] === "escaped") message = message.replace("synthetic-", "\\\\u0073ynthetic-"); const event = process.argv[2] === "codex" ? { type: "turn.failed", error: { message } } : { type: "assistant", error: "invalid_request_error", message: { @@ -25,9 +30,9 @@ const event = process.argv[2] === "codex" console.log(JSON.stringify(event)); process.exitCode = 1; `); -let engine; +let engine, scenario; backend.spawn = (target, spec) => spawn(target, { - ...spec, cmd: process.execPath, args: [cli, engine], + ...spec, cmd: process.execPath, args: [cli, engine, scenario], }); await useFakeRuntime(backend); @@ -39,59 +44,63 @@ const { readEvents } = await import("../src/util/logger.js"); const { getDb } = await import("../src/db/index.js"); for (const selected of ["claude", "codex"]) { - test(`${selected} CLI continuation failures stay readable, redacted and unreplayed`, async () => { - engine = selected; - saveSettings({ engine, engineFallback: false, engineEnabled: { claude: true, codex: true }, composioMode: "personal" }); - await setUser("U_CONTINUATION", { name: "Fixture admin", isAdmin: true, approved: true }); - const channel = `D_CONTINUATION_${engine.toUpperCase()}`; - const entry = await upsertChannelEntry(channel, { name: `continuation-${engine}`, type: "im", isDM: true }); - await saveChannelMeta(entry.slug, { channelId: channel, type: "im", isDM: true, template: "custom", engine, cleanMode: true, allowNetwork: false }); - const posts = [], errors = []; - const client = { chat: { postMessage: async payload => { - posts.push(payload); - return { ok: true, ts: "1900000000.000001" }; - } } }; - const jobs = new BackgroundJobs({ - slack: { snapshot: () => ({ connected: true }), getClient: () => client }, - runner: async args => { - try { return await runMessage({ ...args, preferCold: true }); } - catch (error) { errors.push(error); throw error; } - }, + for (const variant of ["ordinary", "long", "escaped"]) { + test(`${selected} ${variant} CLI continuation failures stay readable, redacted and unreplayed`, async () => { + engine = selected; + scenario = variant; + saveSettings({ engine, engineFallback: false, engineEnabled: { claude: true, codex: true }, composioMode: "personal" }); + await setUser("U_CONTINUATION", { name: "Fixture admin", isAdmin: true, approved: true }); + const channel = `D_CONTINUATION_${engine.toUpperCase()}`; + const entry = await upsertChannelEntry(channel, { name: `continuation-${engine}`, type: "im", isDM: true }); + await saveChannelMeta(entry.slug, { channelId: channel, type: "im", isDM: true, template: "custom", engine, cleanMode: true, allowNetwork: false }); + const posts = [], errors = []; + const client = { chat: { postMessage: async payload => { + posts.push(payload); + return { ok: true, ts: "1900000000.000001" }; + } } }; + const jobs = new BackgroundJobs({ + slack: { snapshot: () => ({ connected: true }), getClient: () => client }, + runner: async args => { + try { return await runMessage({ ...args, preferCold: true }); } + catch (error) { errors.push(error); throw error; } + }, + }); + const rec = { + id: `continuation-${engine}`, kind: "shell", channelId: channel, slug: entry.slug, + authorId: "U_CONTINUATION", threadKey: "1900000000.000002", label: "fixture job", + command: "true", startedAt: Date.now() - 1000, pid: null, maxMs: 60000, + logFile: "", tail: "harmless output", timedOut: false, secretValues: [canary], + }; + jobs.jobs.set(rec.id, rec); + try { + const before = backend.calls.spawn.length; + // Real engine parser and runMessage; only runtime execution and transport are fixtures. + await jobs._finish(rec, { code: 0 }); + assert.equal(backend.calls.spawn.length, before + 1); + assert.equal(errors.length, 1); + assert.equal(errors[0].details.engine, engine); + assert.equal(errors[0].details.providerError, true); + assert.ok(errors[0].details.providerCode); + assert.equal(errors[0].details.exitCode, 1); + const notice = posts.find(post => post.text.includes("couldn't continue"))?.text; + if (variant !== "long") assert.match(notice, /The selected model is unavailable\. \[REDACTED\]/); + assert.doesNotMatch(notice, /\{\s*"error"|fixture_model_unavailable/); + assert.ok(!notice.includes("synthetic-"), "no secret prefix survives truncation or JSON decoding"); + assert.ok(notice.split("couldn't continue: ")[1].length <= 400); + const audit = readEvents({ limit: 200 }).find(row => row.event === "bg_continue_error" && row.id === rec.id); + assert.ok(audit); + assert.ok(!audit.error.includes(canary)); + if (engine === "claude") assert.match(audit.error, /fixture_model_unavailable/, "redacted diagnostic detail is preserved"); + await jobs._deliver(rec); + assert.equal(backend.calls.spawn.length, before + 1, "unknown continuation effects are never replayed"); + assert.ok(posts.some(post => /continuation was interrupted/.test(post.text))); + assert.equal(jobs.jobs.has(rec.id), false); + assert.equal(getDb().prepare("SELECT id FROM bg_jobs WHERE id = ?").get(rec.id), undefined); + assert.ok(backend.calls.leases.every(lease => lease.released)); + } finally { + jobs.jobs.clear(); + getDb().prepare("DELETE FROM bg_jobs WHERE id = ?").run(rec.id); + } }); - const rec = { - id: `continuation-${engine}`, kind: "shell", channelId: channel, slug: entry.slug, - authorId: "U_CONTINUATION", threadKey: "1900000000.000002", label: "fixture job", - command: "true", startedAt: Date.now() - 1000, pid: null, maxMs: 60000, - logFile: "", tail: "harmless output", timedOut: false, secretValues: [canary], - }; - jobs.jobs.set(rec.id, rec); - try { - const before = backend.calls.spawn.length; - // Real engine parser and runMessage; only runtime execution and transport are fixtures. - await jobs._finish(rec, { code: 0 }); - assert.equal(backend.calls.spawn.length, before + 1); - assert.equal(errors.length, 1); - assert.equal(errors[0].details.engine, engine); - assert.equal(errors[0].details.providerError, true); - assert.ok(errors[0].details.providerCode); - assert.equal(errors[0].details.exitCode, 1); - const notice = posts.find(post => post.text.includes("couldn't continue"))?.text; - assert.match(notice, /The selected model is unavailable\. \[REDACTED\]/); - assert.doesNotMatch(notice, /\{\s*"error"|fixture_model_unavailable/); - assert.ok(!notice.includes(canary)); - const audit = readEvents({ limit: 200 }).find(row => row.event === "bg_continue_error" && row.id === rec.id); - assert.ok(audit); - assert.ok(!audit.error.includes(canary)); - if (engine === "claude") assert.match(audit.error, /fixture_model_unavailable/, "redacted diagnostic detail is preserved"); - await jobs._deliver(rec); - assert.equal(backend.calls.spawn.length, before + 1, "unknown continuation effects are never replayed"); - assert.ok(posts.some(post => /continuation was interrupted/.test(post.text))); - assert.equal(jobs.jobs.has(rec.id), false); - assert.equal(getDb().prepare("SELECT id FROM bg_jobs WHERE id = ?").get(rec.id), undefined); - assert.ok(backend.calls.leases.every(lease => lease.released)); - } finally { - jobs.jobs.clear(); - getDb().prepare("DELETE FROM bg_jobs WHERE id = ?").run(rec.id); - } - }); + } }