From bb2b8033ea4413fcdacd28e8e7b527e081705830 Mon Sep 17 00:00:00 2001 From: Maggie Appleton <5599295+MaggieAppleton@users.noreply.github.com> Date: Sat, 3 Oct 2026 07:31:55 +0100 Subject: [PATCH] Add bounded Jev interpreter orchestration --- .../conversation-plan/interpret-batch.test.ts | 62 ++++++ .../conversation-plan/interpret-order.test.ts | 41 ++++ .../conversation-plan/interpret-research.ts | 84 +++++++ .../conversation-plan/interpret-scoring.ts | 23 ++ .../interpret-source.test.ts | 69 ++++++ .../interpret-targeting.test.ts | 113 ++++++++++ .../src/conversation-plan/interpret-types.ts | 31 +++ .../server/src/conversation-plan/interpret.ts | 210 ++++++++++++++++++ .../pipeline-commitment.test-fixtures.ts | 31 +++ .../pipeline-commitment.test.ts | 108 +++++++++ .../pipeline-final-scoped-standalone.test.ts | 73 ++++++ .../pipeline-lifecycle-1.test.ts | 76 +++++++ .../pipeline-lifecycle-3.test.ts | 88 ++++++++ .../pipeline-option-2.test.ts | 123 ++++++++++ .../pipeline-option-4.test.ts | 99 +++++++++ .../pipeline-option-5.test.ts | 129 +++++++++++ .../pipeline-ordinary-save-competing.test.ts | 125 +++++++++++ .../pipeline-ordinary-save.test-fixtures.ts | 140 ++++++++++++ .../pipeline-ordinary-save.test.ts | 134 +++++++++++ .../pipeline-provenance-1.test.ts | 118 ++++++++++ .../pipeline-provenance-2.test.ts | 101 +++++++++ .../pipeline-recommendation-3.test.ts | 117 ++++++++++ .../pipeline-recommendation-4.test.ts | 104 +++++++++ .../pipeline-recommendation-5.test.ts | 87 ++++++++ .../pipeline-recommendation-7.test.ts | 78 +++++++ .../pipeline-recovery-alternatives.test.ts | 91 ++++++++ .../pipeline-recovery-ambiguity.test.ts | 51 +++++ .../pipeline-recovery-refusal.test.ts | 51 +++++ .../pipeline-recovery-repeat.test.ts | 123 ++++++++++ .../pipeline-recovery-unique.test.ts | 76 +++++++ .../pipeline-recovery.test-fixtures.ts | 172 ++++++++++++++ .../pipeline-reply-options.test.ts | 107 +++++++++ .../pipeline-source-fallback.test.ts | 186 ++++++++++++++++ .../pipeline-source-targeting.test.ts | 111 +++++++++ .../question-interpreter.test.ts | 118 ++++++++++ .../recent-thread-context.test.ts | 168 ++++++++++++++ .../research-offers.test-fixtures.ts | 185 +++++++++++++++ 37 files changed, 3803 insertions(+) create mode 100644 apps/server/src/conversation-plan/interpret-batch.test.ts create mode 100644 apps/server/src/conversation-plan/interpret-order.test.ts create mode 100644 apps/server/src/conversation-plan/interpret-research.ts create mode 100644 apps/server/src/conversation-plan/interpret-scoring.ts create mode 100644 apps/server/src/conversation-plan/interpret-source.test.ts create mode 100644 apps/server/src/conversation-plan/interpret-targeting.test.ts create mode 100644 apps/server/src/conversation-plan/interpret-types.ts create mode 100644 apps/server/src/conversation-plan/interpret.ts create mode 100644 apps/server/src/conversation-plan/pipeline-commitment.test-fixtures.ts create mode 100644 apps/server/src/conversation-plan/pipeline-commitment.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-final-scoped-standalone.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-lifecycle-1.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-lifecycle-3.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-option-2.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-option-4.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-option-5.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-ordinary-save-competing.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-ordinary-save.test-fixtures.ts create mode 100644 apps/server/src/conversation-plan/pipeline-ordinary-save.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-provenance-1.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-provenance-2.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recommendation-3.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recommendation-4.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recommendation-5.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recommendation-7.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recovery-alternatives.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recovery-ambiguity.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recovery-refusal.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recovery-repeat.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recovery-unique.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-recovery.test-fixtures.ts create mode 100644 apps/server/src/conversation-plan/pipeline-reply-options.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-source-fallback.test.ts create mode 100644 apps/server/src/conversation-plan/pipeline-source-targeting.test.ts create mode 100644 apps/server/src/conversation-plan/question-interpreter.test.ts create mode 100644 apps/server/src/conversation-plan/recent-thread-context.test.ts create mode 100644 apps/server/src/conversation-plan/research-offers.test-fixtures.ts diff --git a/apps/server/src/conversation-plan/interpret-batch.test.ts b/apps/server/src/conversation-plan/interpret-batch.test.ts new file mode 100644 index 00000000..f78eaac0 --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-batch.test.ts @@ -0,0 +1,62 @@ +import { expect, test } from "bun:test"; +import { interpretMessage } from "./interpret"; +import { initialState } from "./domain"; +import type { JevQuestion } from "./jev"; +import { message, mockResult, seeded, settledBy } from "./interpret.test-fixtures"; + +// Exact archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2, pipeline.test.ts. +// Complete callbacks and helpers retained; injected offline transport only. + +test("a failed target waits for its sibling before the next message starts", async () => { + let current = message( + "failed-batch", + "Sounds good to me. What about agents? Copilot?", + "Jules", + ); + let next = message("next-message", "Should we use GitHub authentication?", "Mina"); + let release!: (result: ReturnType) => void; + let held = new Promise>(resolve => release = resolve); + let heldQuestions!: Record; + let targetCalls = 0; + let nextStarted = false; + let first = interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: settledBy(seeded(), "Mina"), + ask: request => { + if ("new_question" in request.questions) { + return Promise.resolve(mockResult(request.questions, { new_question: 0.95 })); + } + targetCalls++; + if ("c0_role" in request.questions) { + heldQuestions = request.questions; + return held; + } + if ("c1_role" in request.questions) return Promise.reject(new Error("target failed")); + return Promise.resolve(mockResult(request.questions, {})); + }, + }); + let second = first.then(() => + interpretMessage({ + channelId: "channel", + message: next, + recent: [current], + state: initialState(), + ask: async request => { + nextStarted = true; + return mockResult(request.questions, {}); + }, + }) + ); + await Bun.sleep(0); + expect(targetCalls).toBe(3); + expect(nextStarted).toBe(false); + release(mockResult(heldQuestions, {})); + let failed = await first; + await second; + expect(failed.events).toEqual([]); + expect(failed.analysis.status).toBe("failed"); + expect(failed.analysis.passes).toHaveLength(1); + expect(nextStarted).toBe(true); +}); diff --git a/apps/server/src/conversation-plan/interpret-order.test.ts b/apps/server/src/conversation-plan/interpret-order.test.ts new file mode 100644 index 00000000..497616b6 --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-order.test.ts @@ -0,0 +1,41 @@ +import { expect, test } from "bun:test"; +import { postmarkId, sesId, stateWithOptions } from "./research-offers.test-fixtures"; +import { interpretMessage } from "./interpret"; +import { message, mockResult } from "./interpret.test-fixtures"; + +// Preserve original targeting dispatch turn; helper extraction must introduce no extra await. +test.each([false, true])( + "research eligible %s retains targeting dispatch before the next queued observation", + async eligible => { + let targetingCalls = 0; + let observe!: (count: number) => void; + let observed = new Promise(resolve => observe = resolve); + let pending = interpretMessage({ + channelId: "channel", + message: message("dispatch-order", "Should we use GitHub authentication?"), + recent: [], + state: stateWithOptions("auth", "Which authentication?", [ + { id: postmarkId, label: "GitHub" }, + { id: sesId, label: "Email" }, + ]), + ask: request => { + let triage = "new_question" in request.questions; + let research = "research_kind" in request.questions; + if ((triage && !eligible) || research) { + queueMicrotask(() => queueMicrotask(() => observe(targetingCalls))); + } + if (!triage && !research) targetingCalls++; + return Promise.resolve( + mockResult( + request.questions, + triage ? { new_question: 0.95, research_need: eligible ? 0.95 : 0.05 } : {}, + ), + ); + }, + }); + expect(await observed).toBe(1); + let output = await pending; + expect(output.analysis.passes.map(pass => pass.stage)).toEqual(["triage", "targeting"]); + expect(output.researchOffer).toBeUndefined(); + }, +); diff --git a/apps/server/src/conversation-plan/interpret-research.ts b/apps/server/src/conversation-plan/interpret-research.ts new file mode 100644 index 00000000..95b8d3fe --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-research.ts @@ -0,0 +1,84 @@ +import type { Chat } from "@chopin/protocol"; +import type { JevResult } from "./jev"; +import type { InterpretInput, ResearchOfferCandidate } from "./interpret-types"; +import { confidentChoice, noul } from "./interpret-scoring"; +import { MAX_QUOTE_CANDIDATES } from "./quote-budget"; +import type { QuoteCandidate } from "./quotes"; + +export type MemberResearchInput = InterpretInput & { + message: Chat.Entry & { author: Extract }; +}; + +/** The main interpreter retains the original guard, request await and research catch. */ +export function selectResearchOffer( + input: MemberResearchInput, + first: JevResult, + quotes: QuoteCandidate[], + result: JevResult, +): ResearchOfferCandidate | undefined { + let researchOffer: ResearchOfferCandidate | undefined; + if (result.model === first.model) { + let answers = result.answers; + let quoteKey = confidentChoice(answers, "research_quote"); + let threadId = confidentChoice(answers, "research_thread"); + let firstId = confidentChoice(answers, "research_option_a"); + let secondId = confidentChoice(answers, "research_option_b"); + let kind = confidentChoice(answers, "research_kind"); + let selectedQuote = quoteKey?.match(/^q(\d+)$/)?.[1]; + let index = selectedQuote && Number(selectedQuote) < MAX_QUOTE_CANDIDATES + ? selectedQuote + : undefined; + let quote = index === undefined ? undefined : quotes[Number(index)]; + let answered = answers[`research_q${index}_answered`]; + let scope = confidentChoice(answers, `research_q${index}_scope`); + let thread = input.state.threads.find(item => item.id === threadId); + let options = thread?.contributions.filter(item => item.kind === "option") ?? []; + let source = quote && { + messageId: input.message.id, + author: input.message.author, + quote: quote.quote, + start: quote.start, + end: quote.end, + }; + let grounded = quote && thread && source + && ["exploring", "leaning", "reopened"].includes(thread.status) + && noul(first.answers, `c${index}_owned_unretracted`) >= 0.8 + && noul(answers, `research_q${index}_owned`) >= 0.8 + && answered?.type === "noul" && answered.noul <= 0.2 + && /\b(?:costs?|pric(?:e|es|ing)|egress|billing|fees?|rates?)\b/i.test(quote.quote); + if ( + grounded && source && thread + && firstId && secondId && firstId !== secondId + && options.length === 2 + && options.some(item => item.id === firstId) + && options.some(item => item.id === secondId) + && kind === "current-cost-comparison" + ) { + researchOffer = { + source, + threadId: thread.id, + optionIds: [firstId, secondId], + }; + } else if ( + grounded && source && thread && kind === "current-cost-concern" + && [3, 4].includes(options.length) + && input.state.threads.filter(item => + ["exploring", "leaning", "reopened"].includes(item.status) + && [2, 3, 4].includes( + item.contributions.filter(value => value.kind === "option").length, + ) + ).length === 1 + && scope && scope !== "none" + && (scope === "all-current" || options.some(item => item.id === scope)) + ) { + researchOffer = { + kind: "current-cost-concern", + source, + threadId: thread.id, + optionIds: options.map(item => item.id), + ...(scope === "all-current" ? {} : { focusOptionId: scope }), + }; + } + } + return researchOffer; +} diff --git a/apps/server/src/conversation-plan/interpret-scoring.ts b/apps/server/src/conversation-plan/interpret-scoring.ts new file mode 100644 index 00000000..da824637 --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-scoring.ts @@ -0,0 +1,23 @@ +import type { JevAnswer } from "./jev"; + +export function noul(answers: Record, key: string): number { + let answer = answers[key]; + return answer?.type === "noul" ? answer.noul : 0; +} +export function score(answers: Record, key: string): number { + let answer = answers[key]; + return answer?.type === "score" ? answer.score : 0; +} +export function confidentChoice( + answers: Record, + key: string, +): string | undefined { + let answer = answers[key]; + if (answer?.type !== "choice") return; + let ranked = Object.values(answer.probabilities).sort((a, b) => b - a); + if ( + answer.confidence < 0.8 || (ranked[0] ?? 0) < 0.8 + || (ranked[0] ?? 0) - (ranked[1] ?? 0) < 0.2 + ) return; + return answer.choice; +} diff --git a/apps/server/src/conversation-plan/interpret-source.test.ts b/apps/server/src/conversation-plan/interpret-source.test.ts new file mode 100644 index 00000000..87bbf80f --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-source.test.ts @@ -0,0 +1,69 @@ +import { expect, test } from "bun:test"; +import { interpretMessage } from "./interpret"; +import { initialState } from "./domain"; +import type { JevQuestion } from "./jev"; +import { message, mockResult, seeded, settledBy } from "./interpret.test-fixtures"; +import { extractQuotes } from "./quotes"; + +// Exact archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2, pipeline.test.ts. +// Complete callbacks and helpers retained; injected offline transport only. + +test("rejects a fifth source quote before Jev or any partial event", async () => { + let current = message( + "five-quotes", + "Use GitHub OAuth. Use magic links. Send user tokens. Use a GitHub App. Keep credentials separate.", + ); + let calls = 0; + expect(() => extractQuotes(current.text)).toThrow("source quote count exceeds 4"); + + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: initialState(), + ask: async request => { + calls++; + return mockResult(request.questions, {}); + }, + }); + + expect(calls).toBe(0); + expect(output.events).toEqual([]); + expect(output.analysis.status).toBe("failed"); +}); + +test("a late retraction beyond the ownership context fails closed before Jev", async () => { + let calls = 0; + let ask = async (request: { questions: Record }) => { + calls++; + return mockResult(request.questions, {}); + }; + let tooLong = message( + "late-retraction", + `Sounds good to me. ${"x".repeat(4000)} Actually, no—I withdraw that.`, + "Jules", + ); + let blocked = await interpretMessage({ + channelId: "channel", + message: tooLong, + recent: [], + state: settledBy(seeded(), "Mina"), + ask, + }); + expect(blocked.events).toEqual([]); + expect(blocked.analysis).toMatchObject({ + status: "unlinked", + policyGate: "message exceeds ownership context", + passes: [], + }); + expect(calls).toBe(0); + let boundary = message("exact-boundary", `${"x".repeat(3999)}.`, "Jules"); + await interpretMessage({ + channelId: "channel", + message: boundary, + recent: [], + state: initialState(), + ask, + }); + expect(calls).toBe(1); +}); diff --git a/apps/server/src/conversation-plan/interpret-targeting.test.ts b/apps/server/src/conversation-plan/interpret-targeting.test.ts new file mode 100644 index 00000000..1bdc44b0 --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-targeting.test.ts @@ -0,0 +1,113 @@ +import { expect, test } from "bun:test"; +import { interpretMessage } from "./interpret"; +import type { JevQuestion } from "./jev"; +import { message, mockResult, seeded, settledBy, stateWithOption } from "./interpret.test-fixtures"; + +// Exact archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2, pipeline.test.ts. +// Complete callbacks and helpers retained; injected offline transport only. + +test("interprets a declarative pair for a different open topic through both Jev passes", async () => { + let state = stateWithOption( + "Which search service should we use?", + "search-thread", + "existing-search", + "Use PostgreSQL search.", + ); + let current = message("search-alternatives", "By search I mean Algolia or Meilisearch."); + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state, + ask: async request => + mockResult( + request.questions, + calls++ === 0 + ? { new_option: 0.05, thread_target: "search-thread", significance: 2 } + : { + c0_role: "none", + c1_role: "none", + c0_thread: "search-thread", + c1_thread: "search-thread", + c0_new_option: 0.05, + c1_new_option: 0.85, + }, + ), + }); + expect(calls).toBe(3); + expect(output.events.map(event => event.type)).toEqual(["option.added", "option.added"]); + expect(output.events.map(event => "source" in event && event.source)).toMatchObject([ + { quote: "Algolia", start: 17, end: 24, role: "option" }, + { quote: "Meilisearch", start: 28, end: 39, role: "option" }, + ]); +}); + +test("a later target failure or inconsistent model cannot publish partial events", async () => { + let current = message("partial", "Sounds good to me. What about agents? Copilot?", "Jules"); + for (let failure of ["transport", "model"] as const) { + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [message("proposal", "Let's use GitHub.", "Mina")], + state: settledBy(seeded(), "Mina"), + ask: async request => { + calls++; + if (calls === 4 && failure === "transport") throw new Error("synthetic failure"); + let result = mockResult(request.questions, { + new_question: 0.95, + c0_owned_unretracted: 0.95, + c1_owned_unretracted: 0.95, + c2_owned_unretracted: 0.95, + c0_role: "support", + c0_thread: "thread-a", + c0_support: 0.95, + c0_agrees_with_settle: 0.95, + c1_role: "question", + c1_thread: "new", + }); + if (calls === 3 && failure === "model") result.model = "other-model"; + return result; + }, + }); + expect(calls).toBe(4); + expect(output.events).toEqual([]); + expect(output.analysis.status).toBe("failed"); + expect(output.analysis.passes).toHaveLength(1); + } +}); + +test("out-of-order target replies retain their quote-specific answers", async () => { + let current = message( + "out-of-order", + "Sounds good to me. What about agents? Copilot?", + "Jules", + ); + let release: Array<(result: ReturnType) => void> = []; + let requests: Array> = []; + let interpretation = interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: settledBy(seeded(), "Mina"), + ask: request => { + if ("new_question" in request.questions) { + return Promise.resolve(mockResult(request.questions, { new_question: 0.95 })); + } + requests.push(request.questions); + return new Promise(resolve => release.push(resolve)); + }, + }); + await Bun.sleep(0); + expect(requests).toHaveLength(3); + for (let index of [2, 1, 0]) { + release[index]!(mockResult(requests[index]!, { [`c${index}_new_option`]: (index + 1) / 10 })); + } + let output = await interpretation; + expect(output.analysis.passes.map(pass => pass.stage)).toEqual(["triage", "targeting"]); + let answers = output.analysis.passes[1]!.answers; + for (let index = 0; index < 3; index++) { + expect(answers[`c${index}_new_option`]).toEqual({ type: "noul", noul: (index + 1) / 10 }); + } +}); diff --git a/apps/server/src/conversation-plan/interpret-types.ts b/apps/server/src/conversation-plan/interpret-types.ts new file mode 100644 index 00000000..9db393c9 --- /dev/null +++ b/apps/server/src/conversation-plan/interpret-types.ts @@ -0,0 +1,31 @@ +import type { Chat, ConversationPlan } from "@chopin/protocol"; +import type { JevRequest, JevResult } from "./jev"; +import type { LinkedCardOptions } from "./questions"; + +export type Analysis = Omit; +export type ResearchOfferCandidate = { + source: ConversationPlan.ResearchSource; + threadId: string; + optionIds: [string, string]; + kind?: "current-cost-comparison"; +} | { + source: ConversationPlan.ResearchSource; + threadId: string; + optionIds: string[]; + kind: "current-cost-concern"; + focusOptionId?: string; +}; +export type Interpretation = { + events: ConversationPlan.Event[]; + analysis: Analysis; + researchOffer?: ResearchOfferCandidate; +}; +export type InterpretInput = { + channelId: string; + message: Chat.Entry; + recent: readonly Chat.Entry[]; + state: ConversationPlan.State; + linkedCards?: LinkedCardOptions; + /** Injectable transport for domain tests; production defaults to the bounded TypeSafe adapter. */ + ask?: (request: JevRequest) => Promise; +}; diff --git a/apps/server/src/conversation-plan/interpret.ts b/apps/server/src/conversation-plan/interpret.ts new file mode 100644 index 00000000..6e42f9b4 --- /dev/null +++ b/apps/server/src/conversation-plan/interpret.ts @@ -0,0 +1,210 @@ +import type { ConversationPlan } from "@chopin/protocol"; +import { askJev, type JevAnswer } from "./jev"; +import { planEvents } from "./policy"; +import { + buildCandidateTargetingRequest, + buildResearchOfferRequest, + buildTriageRequest, + QUESTION_SET_VERSION, +} from "./questions"; +import { extractQuotes } from "./quotes"; +import { assertQuoteBudget } from "./quote-budget"; +import { noul, score } from "./interpret-scoring"; +import { type MemberResearchInput, selectResearchOffer } from "./interpret-research"; +import type { + Analysis, + Interpretation, + InterpretInput, + ResearchOfferCandidate, +} from "./interpret-types"; +export type { Interpretation, InterpretInput, ResearchOfferCandidate } from "./interpret-types"; + +/** Computes a proposal only; the caller owns fenced persistence and publication. */ +export async function interpretMessage(input: InterpretInput): Promise { + let started = performance.now(); + let passes: ConversationPlan.AnalysisPass[] = []; + let modelVersion = "unavailable"; + let base: Omit = { + questionSetVersion: QUESTION_SET_VERSION, + modelVersion, + passes, + }; + if ( + input.message.streaming || input.message.author.kind === "system" || !input.message.text.trim() + ) { + return { + events: [], + analysis: { ...base, status: "unlinked", policyGate: "ineligible message" }, + }; + } + if (input.message.text.length > 4000) { + return { + events: [], + analysis: { ...base, status: "unlinked", policyGate: "message exceeds ownership context" }, + }; + } + let ask = input.ask ?? askJev; + let researchOffer: ResearchOfferCandidate | undefined; + try { + let quotes = extractQuotes(input.message.text); + assertQuoteBudget(quotes); + let first = await ask(buildTriageRequest( + input.message, + input.recent, + input.state.threads, + quotes, + input.state.events, + )); + modelVersion = first.model; + passes.push({ stage: "triage", answers: first.answers }); + if ( + input.message.author.kind === "member" && quotes.length + && noul(first.answers, "research_need") >= 0.8 + ) { + try { + let result = await ask(buildResearchOfferRequest( + input.message, + input.recent, + input.state.threads, + quotes, + input.state.events, + )); + researchOffer = selectResearchOffer(input as MemberResearchInput, first, quotes, result); + } catch { + // An uncertain or unavailable research judgment never blocks planning analysis. + } + } + let triageTarget = first.answers.thread_target; + let rankedTargets = triageTarget?.type === "choice" + ? Object.values(triageTarget.probabilities).sort((a, b) => b - a) + : []; + let cardTarget = triageTarget?.type === "choice" + && (rankedTargets[0] ?? 0) >= 0.8 + && (rankedTargets[0] ?? 0) - (rankedTargets[1] ?? 0) >= 0.2 + ? triageTarget.choice + : undefined; + let linkedCards = new Map( + cardTarget && input.linkedCards?.has(cardTarget) + ? [[cardTarget, input.linkedCards.get(cardTarget)!]] + : [], + ); + let useful = [ + "new_question", + "new_option", + "reason", + "constraint", + "evidence", + "assumption", + "support", + "objection", + "correction", + "withdrawal", + "explicit_resolution", + "reopening", + ].some((key) => noul(first.answers, key) >= 0.55) + || score(first.answers, "significance") >= 1.5; + if (!useful) { + return { + events: [], + researchOffer, + analysis: { + ...base, + modelVersion, + status: "unlinked", + policyGate: "low planning significance", + latencyMs: Math.round(performance.now() - started), + }, + }; + } + if (!quotes.length) { + return { + events: [], + researchOffer, + analysis: { + ...base, + modelVersion, + status: "unlinked", + policyGate: "no bounded source quote", + latencyMs: Math.round(performance.now() - started), + }, + }; + } + let requests = quotes.map((_, index) => + buildCandidateTargetingRequest( + input.message, + input.recent, + input.state.threads, + quotes, + index, + input.state.events, + linkedCards, + ) + ); + let settled = await Promise.allSettled(requests.map(request => ask(request))); + let targeted = settled.map(result => { + if (result.status === "rejected") throw result.reason; + return result.value; + }); + if (targeted.some(result => result.model !== first.model)) { + throw new Error("inconsistent Jev model versions"); + } + let answers = Object.assign({}, ...targeted.map(result => result.answers)) as Record< + string, + JevAnswer + >; + if (Object.keys(answers).length > 45) throw new Error("too many targeting answers"); + passes.push({ stage: "targeting", answers: structuredClone(answers) }); + let policyAnswers = { ...answers }; + for (let [index, request] of requests.entries()) { + let key = `c${index}_duplicate`; + if (!(key in request.questions)) policyAnswers[key] = { type: "noul", noul: 0 }; + } + let candidates = quotes.map((quote, index) => ({ + ...quote, + answers: Object.fromEntries( + Object.entries(policyAnswers) + .filter(([key]) => key.startsWith(`c${index}_`)) + .map(([key, answer]) => [key.slice(3), answer]), + ), + })); + let planned = planEvents({ + channelId: input.channelId, + message: input.message, + state: input.state, + linkedCards, + first: first.answers, + candidates, + }); + return { + events: planned.events, + researchOffer, + analysis: { + ...base, + modelVersion: first.model, + status: planned.events.length ? "applied" : "unlinked", + selectedTarget: planned.selectedTarget, + quoteValidation: quotes.map((quote) => ({ + start: quote.start, + end: quote.end, + valid: input.message.text.slice(quote.start, quote.end) === quote.quote, + })), + policyGate: planned.policyGate, + outcomes: planned.outcomes, + candidates: planned.candidates, + latencyMs: Math.round(performance.now() - started), + }, + }; + } catch { + return { + events: [], + researchOffer, + analysis: { + ...base, + modelVersion, + status: "failed", + error: "Jev interpretation failed", + latencyMs: Math.round(performance.now() - started), + }, + }; + } +} diff --git a/apps/server/src/conversation-plan/pipeline-commitment.test-fixtures.ts b/apps/server/src/conversation-plan/pipeline-commitment.test-fixtures.ts new file mode 100644 index 00000000..1794d366 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-commitment.test-fixtures.ts @@ -0,0 +1,31 @@ +import type { ConversationPlan } from "@chopin/protocol"; +import { stateWithOption } from "./policy-terminal.test-fixtures"; +import type { MockMutation } from "./pipeline-ordinary-save.test-fixtures"; + +export function hostingState(): ConversationPlan.State { + return stateWithOption( + "Where should the app be hosted?", + "hosting-thread", + "vps-option", + "Host the app on our own VPS.", + ); +} + +export function lowConfidenceResolution(threadId: string, optionId: string): MockMutation { + return (result, prefix) => { + if (!prefix) return; + Object.assign(result.answers[`${prefix}_role`], { + confidence: 0.73, + probabilities: { resolution: 0.76, option: 0.14, none: 0.1 }, + }); + Object.assign(result.answers[`${prefix}_chosen_option`], { + choice: optionId, + confidence: 0.87, + probabilities: { [optionId]: 0.87, new: 0.08, none: 0.05 }, + }); + Object.assign(result.answers[`${prefix}_thread`], { + confidence: 0.99, + probabilities: { [threadId]: 0.99, none: 0.01 }, + }); + }; +} diff --git a/apps/server/src/conversation-plan/pipeline-commitment.test.ts b/apps/server/src/conversation-plan/pipeline-commitment.test.ts new file mode 100644 index 00000000..93bf0585 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-commitment.test.ts @@ -0,0 +1,108 @@ +import { expect, test } from "bun:test"; +import { message } from "./policy-initial.test-fixtures"; +import { addThreadWithOption } from "./policy-terminal.test-fixtures"; +import { + knownChoiceAnswers, + mockInterpret, + newChoiceAnswers, + proposalTriage, +} from "./pipeline-ordinary-save.test-fixtures"; +import { hostingState, lowConfidenceResolution } from "./pipeline-commitment.test-fixtures"; + +test("a noisy Jev commitment can suggest a known option but cannot auto-record", async () => { + let current = message("noisy-vps-commitment", "We've decided to host it on our own VPS."); + let state = hostingState(); + let output = await mockInterpret( + current, + state, + proposalTriage("hosting-thread", "commitment", { explicit_resolution: 0.69 }), + prefix => + knownChoiceAnswers(prefix, "hosting-thread", "vps-option", "resolution", { + chosen_option: 0.87, + explicit_resolution: 0.53, + }), + (result, prefix) => { + if (!prefix) { + result.answers.act.confidence = 0.94; + result.answers.act.probabilities = { commitment: 0.94, proposal: 0.03, other: 0.03 }; + return; + } + lowConfidenceResolution("hosting-thread", "vps-option")(result, prefix); + }, + ); + expect(output.events.map(event => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ + threadId: "hosting-thread", + optionId: "vps-option", + source: { quote: current.text, start: 0, end: current.text.length, role: "resolution" }, + }); + expect(output.events.some(event => event.type === "decision.recorded")).toBe(false); +}); + +test("reported and negated commitment variants never settle", async () => { + let state = hostingState(); + for ( + let [id, text] of [ + ["reported-vps-commitment", "Alice said we've decided to host it on our own VPS."], + ["negated-vps-commitment", "We've decided not to host it on our own VPS."], + ] as const + ) { + let current = message(id, text); + let output = await mockInterpret( + current, + state, + proposalTriage("hosting-thread", "commitment", { explicit_resolution: 0.69 }), + prefix => + knownChoiceAnswers(prefix, "hosting-thread", "vps-option", "resolution", { + explicit_resolution: 0.53, + }), + lowConfidenceResolution("hosting-thread", "vps-option"), + ); + expect(output.events).toEqual([]); + } +}); + +test("attributed commitments cannot trigger the rescue with strong Jev scores", async () => { + let state = hostingState(); + state = addThreadWithOption( + state, + "email-thread", + "Which service should send notification emails?", + "existing-mailgun", + "Use Mailgun for notification emails.", + ); + for ( + let [id, text, target, option] of [ + [ + "told-reported-vps-commitment", + "Alice told me we've decided to host it on our own VPS.", + "hosting-thread", + "vps-option", + ], + [ + "told-reported-new-choice", + "Alice told me let's do Postmark for notification emails.", + "email-thread", + "new", + ], + ] as const + ) { + let current = message(id, text); + let output = await mockInterpret( + current, + state, + proposalTriage(target, "commitment", { + explicit_resolution: 0.98, + new_option: 0.98, + c0_owned_unretracted: 0.95, + }), + prefix => + option === "new" + ? newChoiceAnswers(prefix, target, { explicit_resolution: 0.98 }) + : knownChoiceAnswers(prefix, target, option, "resolution", { + explicit_resolution: 0.98, + }), + ); + expect(output.events).toEqual([]); + } +}); diff --git a/apps/server/src/conversation-plan/pipeline-final-scoped-standalone.test.ts b/apps/server/src/conversation-plan/pipeline-final-scoped-standalone.test.ts new file mode 100644 index 00000000..944f10c6 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-final-scoped-standalone.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from "bun:test"; +import { applyInference } from "./domain"; +import { interpretMessage } from "./interpret"; +import { mockResult } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; +import { d01LinkedEditorCard } from "./pipeline-linked-option.test-fixtures"; + +test("a standalone Lexical spike preference opens a scoped Save and keeps the editor card open", async () => { + let { state, threadId, options, linkedCards } = d01LinkedEditorCard(); + let lexicalId = "01M3QAQWN9TYWMFW3D0EYAZY8H"; + let text = "I’d pick Lexical for the spike"; + let current = message("standalone-lexical-spike", text, "Mei"); + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state, + linkedCards, + ask: async request => { + if ("new_question" in request.questions) { + return mockResult(request.questions, { + new_question: 0.1, + new_option: 0.89, + act: "proposal", + thread_target: threadId, + significance: 2, + support: 0.92, + explicit_resolution: 0.1, + }); + } + let overrides: Record = {}; + for (let key of Object.keys(request.questions)) { + if (key.endsWith("_role")) overrides[key] = "support"; + if (key.endsWith("_thread")) overrides[key] = threadId; + if (/^c\d+_option$/.test(key) || key.endsWith("_chosen_option")) { + overrides[key] = lexicalId; + } + if (key.endsWith("_relation")) overrides[key] = "supports"; + if (key.endsWith("_support")) overrides[key] = 0.92; + if (key.endsWith("_new_option")) overrides[key] = 0.85; + if (key.endsWith("_planning_substance")) overrides[key] = 0.75; + if (key.endsWith("_duplicate")) overrides[key] = 0.05; + } + return mockResult(request.questions, overrides); + }, + }); + let staged = output.events.reduce((next, event) => applyInference(next, event, current), state); + expect(output.events.map(event => event.type)).toEqual(["scoped-choice.proposed"]); + expect(output.events[0]).toMatchObject({ + type: "scoped-choice.proposed", + threadId, + optionId: lexicalId, + label: "Lexical", + scope: "spike", + source: { + messageId: current.id, + quote: text, + start: 0, + end: text.length, + role: "support", + }, + }); + expect(staged.threads[0]!.status).toBe("exploring"); + expect(staged.threads[0]!.pendingScopedChoice).toMatchObject({ + optionId: lexicalId, + label: "Lexical", + scope: "spike", + }); + expect(staged.threads[0]!.decisionHistory).toEqual([]); + expect(staged.threads[0]!.pendingSettle).toBeUndefined(); + expect(staged.threads[0]!.contributions.filter(item => item.kind === "option")).toEqual([]); + expect(linkedCards.get(threadId)?.options).toEqual(options); +}); diff --git a/apps/server/src/conversation-plan/pipeline-lifecycle-1.test.ts b/apps/server/src/conversation-plan/pipeline-lifecycle-1.test.ts new file mode 100644 index 00000000..cd5c6fa3 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-lifecycle-1.test.ts @@ -0,0 +1,76 @@ +import { expect, test } from "bun:test"; +import type { JevQuestion } from "./jev"; +import { planEvents } from "./policy"; +import { buildTargetingRequest } from "./questions"; +import { extractQuotes } from "./quotes"; +import { seeded, seededOptionId } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; +import { decided, discarded } from "./pipeline-lifecycle.test-fixtures"; + +test("a discarded question opens anew only when deliberately raised again", () => { + let state = discarded(seeded()); + let current = message("bring-back", "Should we use an optional outline?"); + let request = buildTargetingRequest(current, [], state.threads, extractQuotes(current.text)); + let choices = (request.questions.c0_thread as Extract).criteria; + expect(choices["thread-a"]).toContain("(discarded)"); + let ask = (raises: number, newQuestion = 0.95) => + planEvents({ + channelId: "channel", + message: current, + state, + first: first({ new_question: newQuestion }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "question", thread: "thread-a" }), + raises_again: { type: "noul", noul: raises }, + }, + }], + }); + expect(ask(0.2).events).toEqual([]); + expect(ask(0.2).outcomes[0].gate).toBe("discarded question not raised again"); + expect(ask(0.9, 0.89).events).toEqual([]); + let raised = ask(0.9); + expect(raised.events.map((event) => event.type)).toEqual(["thread.opened"]); + expect(raised.events[0].threadId).not.toBe("thread-a"); + expect(state.threads[0].status).toBe("discarded"); + let option = message("late-option", "Use a summary instead."); + let blocked = planEvents({ + channelId: "channel", + message: option, + state, + first: first({ new_option: 0.9 }), + candidates: [{ + quote: option.text, + start: 0, + end: option.text.length, + answers: follow({ role: "option", thread: "thread-a" }), + }], + }); + expect(blocked.events).toEqual([]); + expect(blocked.outcomes[0].gate).toBe("thread discarded"); +}); + +test("a decided target cannot receive a settle proposal", () => { + let current = message("closed-settle", "Let's go with the optional outline."); + let output = planEvents({ + channelId: "channel", + message: current, + state: decided(), + first: first({ explicit_resolution: 0.98 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(output.events).toEqual([]); + expect(output.outcomes[0].gate).toBe("settle target needs review"); +}); diff --git a/apps/server/src/conversation-plan/pipeline-lifecycle-3.test.ts b/apps/server/src/conversation-plan/pipeline-lifecycle-3.test.ts new file mode 100644 index 00000000..afdf3821 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-lifecycle-3.test.ts @@ -0,0 +1,88 @@ +import { expect, test } from "bun:test"; +import type { ConversationPlan } from "@chopin/protocol"; +import { applyInference, initialState } from "./domain"; +import { applyEvent } from "./events"; +import { planEvents } from "./policy"; +import { interpretMessage } from "./interpret"; +import { d01RecordedOpening } from "./policy-terminal.test-fixtures"; +import { member, message } from "./policy-initial.test-fixtures"; +import { mockResult, seeded } from "./interpret.test-fixtures"; +import { decided, discarded } from "./pipeline-lifecycle.test-fixtures"; + +test("D01 recovery does not reopen the identical discarded question without a re-raise", () => { + let input = d01RecordedOpening(); + let earlier = message("d01-earlier", input.message.text, "Nia"); + let prior = applyInference(initialState(), { + id: "d01-earlier-open", + type: "thread.opened", + threadId: "d01-earlier-thread", + observedThreadVersion: 0, + origin: "classifier", + actor: { kind: "classifier" }, + at: earlier.ts, + source: { + messageId: earlier.id, + author: earlier.author as ConversationPlan.SourceAuthor, + quote: earlier.text, + start: 0, + end: earlier.text.length, + role: "question", + }, + question: earlier.text, + }, earlier); + let discarded = applyEvent(prior, { + id: "d01-earlier-discard", + type: "thread.discarded", + threadId: "d01-earlier-thread", + observedThreadVersion: prior.threads[0]!.version, + origin: "human", + actor: member("Nia"), + at: earlier.ts + 1, + }); + let repeated = { + ...input, + message: { ...input.message, id: "d01-repeated", ts: earlier.ts + 2 }, + state: discarded, + }; + expect(planEvents(repeated).events).toEqual([]); +}); + +test("multi-option grouping does not revive a discarded thread", async () => { + let state = discarded(seeded()); + let current = message("discarded-options", "Use Auth0. Use custom login."); + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state, + ask: async request => + mockResult( + request.questions, + calls++ === 0 + ? { new_question: 0.97, act: "question", thread_target: "new", significance: 2 } + : { + c0_role: "option", + c0_thread: "thread-a", + c0_new_option: 0.95, + c1_role: "option", + c1_thread: "thread-a", + c1_new_option: 0.95, + }, + ), + }); + expect(output.events).toEqual([]); +}); + +test("ignored chatter leaves accepted earlier decisions intact", async () => { + let state = decided(); + let output = await interpretMessage({ + channelId: "channel", + message: message("m6", "I'll check tomorrow."), + recent: [], + state, + ask: async (request) => mockResult(request.questions, {}), + }); + expect(output.events).toEqual([]); + expect(state.threads[0].decisionHistory).toHaveLength(1); +}); diff --git a/apps/server/src/conversation-plan/pipeline-option-2.test.ts b/apps/server/src/conversation-plan/pipeline-option-2.test.ts new file mode 100644 index 00000000..9cbf515f --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-option-2.test.ts @@ -0,0 +1,123 @@ +import { expect, test } from "bun:test"; +import { planEvents } from "./policy"; +import { extractQuotes } from "./quotes"; +import { seeded, seededOptionId, settledBy, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; +import { discarded } from "./pipeline-lifecycle.test-fixtures"; + +test("a new option remains visible when its wording also sounds like a resolution", () => { + let state = withOption(seeded()); + let current = message( + "new-access-path", + "I guess both. Let's do no harness, just lightweight stuff with API keys for now.", + ); + let quote = "Let's do no harness, just lightweight stuff with API keys for now."; + let start = current.text.indexOf(quote); + let answers = { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.8 }), + role: { + type: "choice" as const, + choice: "resolution", + confidence: 0.44, + probabilities: { resolution: 0.49, option: 0.42, none: 0.09 }, + }, + chosen_option: optionChoice("new"), + new_option: { type: "noul" as const, noul: 0.91 }, + planning_substance: { type: "noul" as const, noul: 0.88 }, + duplicate: { type: "noul" as const, noul: 0.11 }, + }; + let triage = { + ...first({ new_option: 0.86, explicit_resolution: 0.72 }), + significance: { + type: "score" as const, + score: 2.86, + confidence: 0.87, + legend: { "0": "chatter", "1": "minor", "2": "useful", "3": "work" }, + probabilities: { "0": 0, "1": 0, "2": 0.14, "3": 0.86 }, + }, + }; + let assess = ( + candidateAnswers = answers as Record, + firstAnswers = triage as Record, + candidateState = state, + ) => + planEvents({ + channelId: "channel", + message: current, + state: candidateState, + first: firstAnswers, + candidates: [{ quote, start, end: start + quote.length, answers: candidateAnswers }], + }); + let accepted = assess(); + expect(accepted.events.map((event) => event.type)).toEqual(["option.added"]); + expect(accepted.events[0]).toMatchObject({ + threadId: "thread-a", + source: { quote, start, end: start + quote.length, role: "option" }, + contribution: { text: quote, targetId: "thread-a" }, + }); + expect(accepted.events.some((event) => event.type === "settle.suggested")).toBe(false); + expect(assess({ ...answers, chosen_option: optionChoice(seededOptionId) }).events) + .toEqual([]); + expect(assess({ ...answers, new_option: { type: "noul", noul: 0.6 } }).events) + .toEqual([]); + expect(assess({ ...answers, duplicate: { type: "noul", noul: 0.9 } }).events) + .toEqual([]); + expect(assess(answers, { ...triage, new_option: { type: "noul", noul: 0.6 } }).events) + .toEqual([]); + expect(assess(answers, { ...triage, c0_owned_unretracted: { type: "noul", noul: 0.3 } }).events) + .toEqual([]); + expect(assess({ ...answers, thread: optionChoice("none") }).events).toEqual([]); + expect(assess(answers, triage, discarded(state)).events).toEqual([]); +}); + +test("one message can agree with a pending choice and open a separate sourced question", () => { + let current = message( + "combined", + "Sounds good to me. What about agent access? Copilot? BYO API keys?", + "alice", + ); + let state = settledBy(seeded(), "bob"); + let quotes = extractQuotes(current.text); + let candidates = quotes.map((quote, index) => ({ + ...quote, + answers: index === 0 + ? { + ...follow({ role: "support", thread: "thread-a" }), + support: { type: "noul" as const, noul: 0.8 }, + agrees_with_settle: { type: "noul" as const, noul: 0.8 }, + } + : index === 1 + ? follow({ role: "question", thread: "new" }) + : follow({ role: "none", thread: "none" }), + })); + let result = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ new_question: 0.95, support: 0.9 }), + candidates, + }); + expect(result.events.map(event => event.type)).toEqual([ + "stance.changed", + "settle.agreed", + "thread.opened", + ]); + expect(result.events.filter(event => event.type === "decision.recorded")).toEqual([]); + expect(result.events.map(event => "source" in event ? event.source?.quote : undefined)) + .toEqual([quotes[0]?.quote, quotes[0]?.quote, quotes[1]?.quote]); + let uncertain = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ new_question: 0.95, support: 0.9 }), + candidates: [{ + ...candidates[0]!, + answers: { + ...candidates[0]!.answers, + thread: follow({ role: "support", thread: "none" }).thread, + }, + }, ...candidates.slice(1)], + }); + expect(uncertain.events.map(event => event.type)).toEqual(["thread.opened"]); +}); diff --git a/apps/server/src/conversation-plan/pipeline-option-4.test.ts b/apps/server/src/conversation-plan/pipeline-option-4.test.ts new file mode 100644 index 00000000..cd930596 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-option-4.test.ts @@ -0,0 +1,99 @@ +import { expect, test } from "bun:test"; +import type { Chat } from "@chopin/protocol"; +import { planEvents } from "./policy"; +import { seeded, seededOptionId, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; + +test("the Planner can quote an option but cannot propose a settle", () => { + let state = withOption(seeded()); + let current: Chat.Entry = { + ...message("agent-settle", "Let's go with the optional outline."), + author: { kind: "agent" }, + }; + let base = { + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.98 }), + }; + let settle = planEvents({ + ...base, + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(settle.events).toEqual([]); + let option = { ...current, id: "agent-option", text: "Try a managed provider." }; + let quoted = planEvents({ + ...base, + message: option, + first: first({ new_option: 0.98 }), + candidates: [{ + quote: option.text, + start: 0, + end: option.text.length, + answers: follow({ role: "option", thread: "thread-a" }), + }], + }); + expect(quoted.events.map((event) => event.type)).toEqual(["option.added"]); + expect(quoted.events[0]).toMatchObject({ origin: "planner" }); + let addition = quoted.events[0]; + expect(addition.type === "option.added" && addition.contribution.id) + .toMatch(/^[0-7][0-9A-HJKMNP-TV-Z]{25}$/); +}); + +test("a settle with no known chosen option stays in review", () => { + let current = message("new-choice", "Let's just go with something new."); + let output = planEvents({ + channelId: "channel", + message: current, + state: withOption(seeded()), + first: first({ explicit_resolution: 0.98 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + chosen_option: optionChoice("new"), + }, + }], + }); + expect(output.events).toEqual([]); + expect(output.outcomes[0]).toMatchObject({ + status: "review", + gate: "chosen option needs review", + }); +}); + +test("negated proposals cannot suggest a rejected option", () => { + let state = withOption(seeded()); + let current = message("negated", "We've decided not to use the optional outline."); + for (let chosen of ["none", seededOptionId]) { + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.98 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + option: optionChoice(seededOptionId), + chosen_option: optionChoice(chosen), + }, + }], + }); + expect(output.events).toEqual([]); + expect(output.outcomes[0].gate).toBe("chosen option needs review"); + } +}); diff --git a/apps/server/src/conversation-plan/pipeline-option-5.test.ts b/apps/server/src/conversation-plan/pipeline-option-5.test.ts new file mode 100644 index 00000000..1a6d6074 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-option-5.test.ts @@ -0,0 +1,129 @@ +import { expect, test } from "bun:test"; +import type { ConversationPlan } from "@chopin/protocol"; +import { applyInference } from "./domain"; +import { planEvents } from "./policy"; +import { seeded, seededOptionId, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; + +test("an unclear choice is held, but current opposition does not block a clear suggestion", () => { + let state = withOption(seeded()); + let opposition = message("oppose-option", "I oppose the optional outline.", "Theo"); + state = applyInference(state, { + id: "opposition-option", + type: "stance.changed", + scopedProposalId: null, + threadId: "thread-a", + observedThreadVersion: state.threads[0].version, + origin: "classifier", + actor: { kind: "classifier" }, + at: 1000, + source: { + messageId: opposition.id, + author: opposition.author as ConversationPlan.SourceAuthor, + quote: opposition.text, + start: 0, + end: opposition.text.length, + role: "objection", + }, + optionId: seededOptionId, + position: "oppose", + }, opposition); + let current = message("uncertain-choice", "Let's go with the optional outline."); + let input = { + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.98 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + chosen_option: optionChoice("none"), + }, + }], + }; + let unclear = planEvents(input); + expect(unclear.events).toEqual([]); + expect(unclear.outcomes[0].gate).toBe("chosen option needs review"); + let clear = planEvents({ + ...input, + candidates: [{ + ...input.candidates[0], + answers: { + ...input.candidates[0].answers, + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(clear.events.map((event) => event.type)).toEqual(["settle.suggested"]); +}); + +test("proposal, assent, quote, sarcasm, and reported consensus do not decide", () => { + let state = seeded(); + for ( + let [id, text] of [ + ["proposal", "Maybe use an optional outline."], + ["assent", "+1 to the optional outline."], + ["quote", "Please show ‘we decided to use an outline’ in history."], + ["sarcasm", "Sure, force everyone through a template 🙄"], + ["reported", "Everyone seems to agree on an outline, I guess."], + ] + ) { + let current = message(id, text); + let result = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.05 }), + candidates: [ + { + quote: text, + start: 0, + end: text.length, + answers: follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.04 }), + }, + ], + }); + expect(result.events).toEqual([]); + } +}); + +test("a settle needs strong first-pass and candidate evidence", () => { + let state = withOption(seeded()); + let current = message("resolution", "We've decided to use the optional outline."); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.79 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + }], + }); + expect(output.events).toEqual([]); + expect(output.outcomes[0].gate).toBe("settle authority unclear"); + let weakCandidate = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.98 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.84 }), + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(weakCandidate.events).toEqual([]); + expect(weakCandidate.outcomes[0].gate).toBe("settle authority unclear"); + expect(state.threads[0].status).toBe("exploring"); +}); diff --git a/apps/server/src/conversation-plan/pipeline-ordinary-save-competing.test.ts b/apps/server/src/conversation-plan/pipeline-ordinary-save-competing.test.ts new file mode 100644 index 00000000..b5185e36 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-ordinary-save-competing.test.ts @@ -0,0 +1,125 @@ +import { expect, test } from "bun:test"; +import { applyInference } from "./domain"; +import { extractQuotes } from "./quotes"; +import { message } from "./policy-initial.test-fixtures"; +import { + emailState, + knownChoiceAnswers, + mockInterpret, + newChoiceAnswers, + proposalTriage, +} from "./pipeline-ordinary-save.test-fixtures"; +import { queueWithPendingSave } from "./pipeline-ordinary-save-pending.test-fixtures"; + +// Whole archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2 pipeline.test.ts callbacks. +test("distinct new choices for one thread add sourced options without choosing between them", async () => { + let current = message( + "competing-same-thread", + "We should use Postmark for notification emails. We should use Amazon SES for notification emails.", + ); + let quotes = extractQuotes(current.text); + let output = await mockInterpret( + current, + emailState(), + proposalTriage("email-thread"), + prefix => newChoiceAnswers(prefix, "email-thread"), + ); + expect(output.events.map(event => event.type)).toEqual(["option.added", "option.added"]); + expect(output.events.map(event => event.type === "option.added" ? event.source?.quote : "")) + .toEqual(quotes.map(item => item.quote)); + for (let [index, quote] of quotes.entries()) { + expect(output.events[index]).toMatchObject({ + threadId: "email-thread", + contribution: { text: quote.quote }, + source: { quote: quote.quote, start: quote.start, end: quote.end, role: "option" }, + }); + } + expect(output.analysis.outcomes?.[1]).toMatchObject({ + start: quotes[1]!.start, + end: quotes[1]!.end, + status: "review", + }); +}); + +test("a pending Save survives later competing known and new choices", async () => { + let { state, proposal } = queueWithPendingSave(); + for ( + let item of [ + { + id: "later-known-choice", + text: "I think we should use the PostgreSQL queue.", + newChoice: false, + optionId: "postgres-option", + }, + { + id: "later-new-choice", + text: "We should use BullMQ for background jobs.", + newChoice: true, + optionId: "new", + }, + ] as const + ) { + let current = message(item.id, item.text, "Omar"); + let triage = proposalTriage("queue-thread", "proposal", { + new_option: item.newChoice ? 0.98 : 0.1, + }); + let output = await mockInterpret( + current, + state, + triage, + prefix => + item.newChoice + ? newChoiceAnswers(prefix, "queue-thread") + : knownChoiceAnswers(prefix, "queue-thread", item.optionId), + ); + expect(output.events).toEqual([]); + expect(output.analysis.outcomes?.[0]).toMatchObject({ + start: 0, + end: current.text.length, + status: "review", + }); + let next = output.events.reduce( + (currentState, event) => applyInference(currentState, event, current), + state, + ); + expect(next.threads.find(thread => thread.id === "queue-thread")?.pendingSettle) + .toEqual({ optionId: "sqs-option", proposer: "Mina", messageId: proposal.id }); + } +}); + +test("repeating the same new choice twice opens one Save suggestion", async () => { + let text = "We should use Postmark for notification emails."; + let current = message("repeat-same-choice", `${text} ${text}`); + let quotes = extractQuotes(current.text); + let output = await mockInterpret( + current, + emailState(), + proposalTriage("email-thread"), + prefix => newChoiceAnswers(prefix, "email-thread"), + ); + expect(quotes.map(item => item.quote)).toEqual([text, text]); + expect(output.events.map(event => event.type)).toEqual(["option.added", "settle.suggested"]); + let added = output.events[0]; + let selectedSource = added && "source" in added ? added.source : undefined; + expect(added).toMatchObject({ + type: "option.added", + threadId: "email-thread", + contribution: { text }, + }); + expect( + quotes.some(quote => + quote.start === selectedSource?.start && quote.end === selectedSource?.end + && quote.quote === selectedSource?.quote + ), + ).toBe(true); + expect(output.events[1]).toMatchObject({ + type: "settle.suggested", + optionId: added?.type === "option.added" ? added.contribution.id : undefined, + source: { + quote: selectedSource?.quote, + start: selectedSource?.start, + end: selectedSource?.end, + role: "resolution", + }, + }); +}); diff --git a/apps/server/src/conversation-plan/pipeline-ordinary-save.test-fixtures.ts b/apps/server/src/conversation-plan/pipeline-ordinary-save.test-fixtures.ts new file mode 100644 index 00000000..5c48c6d3 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-ordinary-save.test-fixtures.ts @@ -0,0 +1,140 @@ +import { expect } from "bun:test"; +import type { Chat, ConversationPlan } from "@chopin/protocol"; +import { interpretMessage } from "./interpret"; +import { mockResult, seededOptionId } from "./interpret.test-fixtures"; +import { stateWithOption } from "./policy-terminal.test-fixtures"; + +// Exact archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2 pipeline.test.ts declarations. +export type MockAnswers = Record; + +export type MockMutation = (result: any, prefix?: string) => void; + +export function emailState(): ConversationPlan.State { + return stateWithOption( + "Which service should send notification emails?", + "email-thread", + "existing-mailgun", + "We should use Mailgun for notification emails.", + ); +} + +export function proposalTriage( + target: string, + act = "proposal", + overrides: MockAnswers = {}, +): MockAnswers { + return { + new_question: 0.1, + new_option: 0.98, + act, + thread_target: target, + significance: 2, + explicit_resolution: 0.1, + ...overrides, + }; +} + +export function newChoiceAnswers( + prefix: string, + target: string, + overrides: MockAnswers = {}, +): MockAnswers { + return withCandidateOverrides(prefix, { + [`${prefix}_role`]: "resolution", + [`${prefix}_thread`]: target, + [`${prefix}_option`]: "new", + [`${prefix}_chosen_option`]: "new", + [`${prefix}_new_option`]: 0.98, + [`${prefix}_planning_substance`]: 0.96, + [`${prefix}_support`]: 0.96, + [`${prefix}_explicit_resolution`]: 0.1, + [`${prefix}_duplicate`]: 0.04, + }, overrides); +} + +export function knownChoiceAnswers( + prefix: string, + target: string, + option: string, + role = "resolution", + overrides: MockAnswers = {}, +): MockAnswers { + return withCandidateOverrides(prefix, { + [`${prefix}_role`]: role, + [`${prefix}_thread`]: target, + [`${prefix}_option`]: option, + [`${prefix}_chosen_option`]: option, + [`${prefix}_new_option`]: 0.05, + [`${prefix}_planning_substance`]: 0.05, + [`${prefix}_support`]: 0.96, + [`${prefix}_explicit_resolution`]: 0.1, + [`${prefix}_duplicate`]: 0.04, + }, overrides); +} + +export function withCandidateOverrides( + prefix: string, + answers: MockAnswers, + overrides: MockAnswers, +): MockAnswers { + for (let [key, value] of Object.entries(overrides)) { + answers[key.startsWith(`${prefix}_`) ? key : `${prefix}_${key}`] = value; + } + return answers; +} + +export async function mockInterpret( + current: Chat.Entry, + state: ConversationPlan.State, + triage: MockAnswers, + candidate: (prefix: string) => MockAnswers, + mutate?: MockMutation, + recent: readonly Chat.Entry[] = [], +) { + return interpretMessage({ + channelId: "channel", + message: current, + recent, + state, + ask: async request => { + let isTriage = "new_question" in request.questions; + let prefix = Object.keys(request.questions).find(key => /^c\d+_role$/.test(key)) + ?.replace(/_role$/, ""); + let result = mockResult(request.questions, isTriage ? triage : candidate(prefix ?? "c0")); + mutate?.(result, isTriage ? undefined : prefix); + return result; + }, + }); +} + +export function expectSourcedOptionSave( + events: ConversationPlan.Event[], + threadId: string, + quote: { quote: string; start: number; end: number }, +): void { + expect(events.map(event => event.type)).toEqual(["option.added", "settle.suggested"]); + let added = events[0]; + expect(added).toMatchObject({ + type: "option.added", + threadId, + contribution: { text: quote.quote }, + source: { ...quote, role: "option" }, + }); + expect(events[1]).toMatchObject({ + type: "settle.suggested", + threadId, + optionId: added?.type === "option.added" ? added.contribution.id : undefined, + source: { ...quote, role: "resolution" }, + }); +} + +export function optionChoice(choice: string): any { + return { + type: "choice", + choice, + confidence: 0.95, + probabilities: Object.fromEntries( + [seededOptionId, "new", "none"].map((key) => [key, key === choice ? 0.96 : 0.02]), + ), + }; +} diff --git a/apps/server/src/conversation-plan/pipeline-ordinary-save.test.ts b/apps/server/src/conversation-plan/pipeline-ordinary-save.test.ts new file mode 100644 index 00000000..1f768dde --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-ordinary-save.test.ts @@ -0,0 +1,134 @@ +import { expect, test } from "bun:test"; +import { planEvents } from "./policy"; +import { extractQuotes } from "./quotes"; +import { seeded, seededOptionId, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { addThreadWithOption } from "./policy-terminal.test-fixtures"; +import { + emailState, + expectSourcedOptionSave, + mockInterpret, + newChoiceAnswers, + optionChoice, + proposalTriage, +} from "./pipeline-ordinary-save.test-fixtures"; + +// Whole archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2 pipeline.test.ts callbacks. +test("a known choice becomes a settle suggestion, never a decision", () => { + let current = message("m2", "Let's just go with the optional outline."); + let state = withOption(seeded()); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.98 }), + candidates: [ + { + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + chosen_option: optionChoice(seededOptionId), + }, + }, + ], + }); + expect(output.events.map((event) => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ optionId: seededOptionId }); + expect("source" in output.events[0] ? output.events[0].source?.quote : undefined).toBe( + current.text, + ); + expect(state.threads[0].status).toBe("exploring"); + let ambiguous = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.98 }), + candidates: [ + { + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ + role: "resolution", + thread: "thread-a", + threadProbability: 0.65, + explicit_resolution: 0.98, + }), + chosen_option: optionChoice(seededOptionId), + }, + }, + ], + }); + expect(ambiguous.events).toEqual([]); +}); + +test.each( + [ + { + id: "new-postmark-choice", + text: "We should use Postmark for notification emails.", + act: "proposal", + }, + { + id: "new-postmark-commitment", + text: "Let's do Postmark for notification emails.", + act: "commitment", + }, + { + id: "new-postmark-decided-choice", + text: "We've decided to use Postmark for notification emails.", + act: "commitment", + }, + ] as const, +)("a new choice in $act text stages it and suggests Save", async ({ id, text, act }) => { + let current = message(id, text); + let state = emailState(); + let quote = extractQuotes(text)[0]!; + let output = await mockInterpret( + current, + state, + proposalTriage("email-thread", act, { explicit_resolution: 0.94 }), + prefix => newChoiceAnswers(prefix, "email-thread", { explicit_resolution: 0.94 }), + ); + expectSourcedOptionSave(output.events, "email-thread", quote); + expect(output.events.some(event => event.type === "decision.recorded")).toBe(false); +}); + +test("direct choices for distinct threads both suggest Save", async () => { + let current = message( + "two-thread-proposals", + "We should use Postmark for notifications. We should use Redis for background jobs.", + ); + let state = emailState(); + state = addThreadWithOption( + state, + "jobs-thread", + "Should background jobs use Redis or a PostgreSQL queue?", + "existing-postgres-queue", + "Use a PostgreSQL queue.", + ); + let quotes = extractQuotes(current.text); + let output = await mockInterpret( + current, + state, + proposalTriage("none"), + prefix => newChoiceAnswers(prefix, prefix === "c0" ? "email-thread" : "jobs-thread"), + ); + expect(output.events.map(event => event.type)).toEqual([ + "option.added", + "settle.suggested", + "option.added", + "settle.suggested", + ]); + for (let [index, quote] of quotes.entries()) { + expectSourcedOptionSave( + output.events.slice(index * 2, index * 2 + 2), + index === 0 ? "email-thread" : "jobs-thread", + quote, + ); + } + expect(output.events.some(event => event.type === "decision.recorded")).toBe(false); +}); diff --git a/apps/server/src/conversation-plan/pipeline-provenance-1.test.ts b/apps/server/src/conversation-plan/pipeline-provenance-1.test.ts new file mode 100644 index 00000000..ff6462b3 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-provenance-1.test.ts @@ -0,0 +1,118 @@ +import { expect, test } from "bun:test"; +import { planEvents } from "./policy"; +import { extractQuotes } from "./quotes"; +import { seeded, seededOptionId, settledBy } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; + +test("another member's plain assent agrees without recording a decision", () => { + let state = settledBy(seeded(), "bob"); + for ( + let [id, text, role] of [ + ["assent-support", "Sounds good to me.", "support"], + ["assent-none", "Sounds good to me.", "none"], + ["seems-good", "Seems good to me.", "none"], + ["no-reason-not", "Sounds good—no reason not to.", "support"], + ] + ) { + let current = message(id, text, "alice"); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ support: 0.9 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role, thread: "thread-a" }), + agrees_with_settle: { type: "noul", noul: 0.9 }, + }, + }], + }); + expect(output.events.map((event) => event.type)).toEqual( + role === "support" ? ["stance.changed", "settle.agreed"] : ["settle.agreed"], + ); + expect(output.events.at(-1)).toMatchObject({ + type: "settle.agreed", + optionId: seededOptionId, + origin: "classifier", + }); + expect(state.threads[0].decision).toBeUndefined(); + } +}); + +test("a later attribution vetoes only the earlier assent, retaining objection and question", () => { + let state = settledBy(seeded(), "Bob"); + let current = message( + "mixed-withdrawal", + "Sounds good to me. That's what Bob said, but I disagree. What about agent access?", + "Jules", + ); + let quotes = extractQuotes(current.text); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ + new_question: 0.95, + c0_owned_unretracted: 0.2, + c1_owned_unretracted: 0.9, + c2_owned_unretracted: 0.9, + }), + candidates: [ + { + ...quotes[0]!, + answers: { + ...follow({ role: "support", thread: "thread-a" }), + agrees_with_settle: { type: "noul", noul: 0.95 }, + }, + }, + { ...quotes[1]!, answers: follow({ role: "objection", thread: "thread-a" }) }, + { ...quotes[2]!, answers: follow({ role: "question", thread: "new" }) }, + ], + }); + expect(output.events.map(event => event.type)).toEqual(["stance.changed", "thread.opened"]); + expect(output.events[0]).toMatchObject({ position: "oppose" }); + expect(output.events.map(event => "source" in event ? event.source?.quote : "")) + .toEqual([quotes[1]!.quote, quotes[2]!.quote]); + expect(output.outcomes[0]?.gate).toBe("source ownership unclear"); + expect(output.events.some(event => + event.type === "settle.agreed" + || event.type === "settle.suggested" || event.type === "decision.recorded" + )).toBe(false); +}); + +test("direct support for the pending option agrees even when generic agreement is low", () => { + let state = settledBy(seeded(), "bob"); + for ( + let [id, text] of [ + ["direct-support", "I support the optional outline."], + ["qualified-support", "I don't see a reason not to use the optional outline."], + ] + ) { + let current = message(id, text, "alice"); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ support: 0.95 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "support", thread: "thread-a" }), + option: optionChoice(seededOptionId), + agrees_with_settle: { type: "noul", noul: 0.1 }, + }, + }], + }); + expect(output.events.map((event) => event.type)).toEqual([ + "stance.changed", + "settle.agreed", + ]); + expect(output.events[1]).toMatchObject({ optionId: seededOptionId }); + } +}); diff --git a/apps/server/src/conversation-plan/pipeline-provenance-2.test.ts b/apps/server/src/conversation-plan/pipeline-provenance-2.test.ts new file mode 100644 index 00000000..31487438 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-provenance-2.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from "bun:test"; +import { planEvents } from "./policy"; +import { extractQuotes } from "./quotes"; +import { seeded, seededOptionId, settledBy, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; + +test("reported, sarcastic, and retracted quotes cannot borrow strong agreement evidence", () => { + for ( + let [text, owned] of [ + ["Bob said, 'sounds good to me.'", 0.04], + ["Sounds good to me 🙄. What about agents?", 0.22], + ["Sounds good to me. Actually, no—I disagree with GitHub.", 0.17], + ] as const + ) { + let current = message(`negative-${owned}`, text, "Jules"); + let quote = extractQuotes(current.text)[0]!; + let output = planEvents({ + channelId: "channel", + message: current, + state: settledBy(seeded(), "Mina"), + first: first({ support: 0.95, c0_owned_unretracted: owned }), + candidates: [{ + ...quote, + answers: { + ...follow({ role: "support", thread: "thread-a" }), + agrees_with_settle: { type: "noul", noul: 0.95 }, + }, + }], + }); + expect(output.events).toEqual([]); + expect(output.outcomes[0]?.gate).toBe("source ownership unclear"); + } + let current = message("retracted-resolution", "Let's use Redis. Actually, no.", "Jules"); + let quote = extractQuotes(current.text)[0]!; + let output = planEvents({ + channelId: "channel", + message: current, + state: withOption(seeded()), + first: first({ explicit_resolution: 0.98, c0_owned_unretracted: 0.09 }), + candidates: [{ + ...quote, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.98 }), + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(output.events).toEqual([]); + expect(output.outcomes[0]?.gate).toBe("source ownership unclear"); +}); + +test("a separately sourced concern and question can follow assent in one message", () => { + let state = settledBy(seeded(), "Mina"); + let other = structuredClone(state.threads[0]!); + other.id = "thread-b"; + other.question = "How should we handle agent access?"; + other.pendingSettle = undefined; + state.threads.push(other); + let current = message( + "mixed-three", + "Sounds good to me. I'm worried about agent access. Should we use Copilot?", + "Jules", + ); + let quotes = extractQuotes(current.text); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ + new_question: 0.95, + c0_owned_unretracted: 0.9, + c1_owned_unretracted: 0.9, + c2_owned_unretracted: 0.9, + }), + candidates: [ + { + ...quotes[0]!, + answers: { + ...follow({ role: "support", thread: "thread-a" }), + agrees_with_settle: { type: "noul", noul: 0.95 }, + }, + }, + { ...quotes[1]!, answers: follow({ role: "objection", thread: "thread-b" }) }, + { ...quotes[2]!, answers: follow({ role: "question", thread: "new" }) }, + ], + }); + expect(output.events.map(event => event.type)).toEqual([ + "stance.changed", + "settle.agreed", + "stance.changed", + "thread.opened", + ]); + expect(output.events.map(event => "source" in event ? event.source?.quote : "")) + .toEqual([quotes[0]!.quote, quotes[0]!.quote, quotes[1]!.quote, quotes[2]!.quote]); + expect( + output.events.every(event => "source" in event && event.source?.messageId === current.id), + ) + .toBe(true); + expect(output.events.some(event => event.type === "decision.recorded")).toBe(false); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recommendation-3.test.ts b/apps/server/src/conversation-plan/pipeline-recommendation-3.test.ts new file mode 100644 index 00000000..8b9b57a4 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recommendation-3.test.ts @@ -0,0 +1,117 @@ +import { expect, test } from "bun:test"; +import { message } from "./policy-initial.test-fixtures"; +import { addThreadWithOption, stateWithOption } from "./policy-terminal.test-fixtures"; +import { + emailState, + knownChoiceAnswers, + mockInterpret, + newChoiceAnswers, + proposalTriage, +} from "./pipeline-ordinary-save.test-fixtures"; + +test("weak, hedged, negative, reported, duplicate, and list wording cannot preselect", async () => { + for ( + let item of [ + { id: "postmark-guess", text: "We should use Postmark for emails, I guess.", empty: false }, + { + id: "postmark-probably", + text: "We should use Postmark for emails, probably.", + empty: false, + }, + { + id: "hedged-new-choice", + text: "Maybe we should use Postmark for notification emails.", + empty: true, + }, + { + id: "negated-new-choice", + text: "We should not use Postmark for notification emails.", + empty: true, + }, + { + id: "reported-new-choice", + text: "Alice said we should use Postmark for notification emails.", + empty: true, + }, + { + id: "duplicate-new-choice", + text: "We should use Mailgun for notification emails.", + empty: true, + }, + { + id: "plain-option-list", + text: "Postmark or Amazon SES for notification emails.", + empty: false, + }, + ] as const + ) { + let current = message(item.id, item.text); + let duplicate = item.id === "duplicate-new-choice" ? 0.96 : 0.04; + let output = await mockInterpret( + current, + emailState(), + proposalTriage("email-thread", "proposal", { duplicate }), + prefix => + newChoiceAnswers(prefix, "email-thread", { + role: item.id === "plain-option-list" ? "option" : "resolution", + duplicate, + }), + ); + expect(output.events.some(event => event.type === "settle.suggested")).toBe(false); + expect(output.events.some(event => event.type === "decision.recorded")).toBe(false); + if (item.empty) expect(output.events).toEqual([]); + } +}); + +test("ambiguous thread targeting cannot settle a direct new choice", async () => { + let state = emailState(); + state = addThreadWithOption( + state, + "other-email-thread", + "Should we use a different email provider?", + "existing-ses", + "Use Amazon SES.", + ); + let current = message( + "ambiguous-new-choice", + "We should use Postmark for notification emails.", + ); + let output = await mockInterpret( + current, + state, + proposalTriage("none"), + prefix => newChoiceAnswers(prefix, "none"), + (result, prefix) => { + if (!prefix) return; + let target = result.answers[`${prefix}_thread`]; + target.choice = "none"; + target.confidence = 0.5; + target.probabilities = { "email-thread": 0.47, "other-email-thread": 0.46, none: 0.07 }; + }, + ); + expect(output.events).toEqual([]); +}); + +test("I think we should use a known option suggests Save, not a decision", async () => { + let current = message("existing-option-recommendation", "I think we should use SQS."); + let state = stateWithOption( + "Which queue should handle background jobs?", + "queue-thread", + "sqs-option", + "Use SQS.", + ); + let output = await mockInterpret( + current, + state, + proposalTriage("queue-thread", "proposal", { new_option: 0.1 }), + prefix => knownChoiceAnswers(prefix, "queue-thread", "sqs-option", "support"), + ); + expect(output.events.map(event => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ + threadId: "queue-thread", + optionId: "sqs-option", + source: { quote: current.text, start: 0, end: current.text.length, role: "resolution" }, + }); + expect(output.events.some(event => event.type === "option.added")).toBe(false); + expect(output.events.some(event => event.type === "decision.recorded")).toBe(false); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recommendation-4.test.ts b/apps/server/src/conversation-plan/pipeline-recommendation-4.test.ts new file mode 100644 index 00000000..c7296787 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recommendation-4.test.ts @@ -0,0 +1,104 @@ +import { expect, test } from "bun:test"; +import { planEvents } from "./policy"; +import { seeded, seededOptionId, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; + +test("a clear owned team commitment suggests a known choice despite low settle scores", () => { + let state = withOption(seeded()); + state.threads[0]!.question = "Where should we host the app?"; + state.threads[0]!.contributions[0]!.text = "Host it on our own VPS"; + let current = message("vps-choice", "We've decided to host it on our own VPS"); + let input = { + channelId: "channel", + message: current, + state, + first: { + ...first({ explicit_resolution: 0.60, c0_owned_unretracted: 0.89 }), + act: { + type: "choice" as const, + choice: "commitment", + confidence: 0.89, + probabilities: { commitment: 0.89, proposal: 0.08, other: 0.03 }, + }, + }, + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.67 }), + role: { + type: "choice" as const, + choice: "resolution", + confidence: 0.89, + probabilities: { resolution: 0.89, support: 0.08, none: 0.03 }, + }, + chosen_option: { + type: "choice" as const, + choice: seededOptionId, + confidence: 0.92, + probabilities: { [seededOptionId]: 0.92, new: 0.04, none: 0.04 }, + }, + }, + }], + }; + let output = planEvents(input); + expect(output.events.map(event => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ + optionId: seededOptionId, + source: { quote: current.text, messageId: current.id }, + }); + expect(state.threads[0].status).toBe("exploring"); + let uncertainAct = planEvents({ + ...input, + first: { + ...input.first, + act: { + type: "choice", + choice: "proposal", + confidence: 0.89, + probabilities: { proposal: 0.89, commitment: 0.08, other: 0.03 }, + }, + }, + }); + expect(uncertainAct.events).toEqual([]); + expect(uncertainAct.outcomes[0]?.gate).toBe("settle authority unclear"); +}); + +test("a recommendation for a known VPS option does not add a duplicate", () => { + let state = withOption(seeded()); + state.threads[0]!.question = "Where should we host the app?"; + state.threads[0]!.contributions[0]!.text = "Host it on our own VPS"; + let current = message("vps-proposal", "We should also host it on our own VPS"); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: { + ...first({ explicit_resolution: 0.45, new_option: 0.82 }), + act: { + type: "choice", + choice: "proposal", + confidence: 1, + probabilities: { proposal: 1, commitment: 0 }, + }, + }, + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "option", thread: "thread-a", explicit_resolution: 0.67 }), + option: optionChoice(seededOptionId), + chosen_option: optionChoice(seededOptionId), + new_option: { type: "noul", noul: 0.91 }, + duplicate: { type: "noul", noul: 0.53 }, + support: { type: "noul", noul: 0.90 }, + }, + }], + }); + expect(output.events.map(event => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ optionId: seededOptionId }); + expect(state.threads[0]!.contributions).toHaveLength(1); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recommendation-5.test.ts b/apps/server/src/conversation-plan/pipeline-recommendation-5.test.ts new file mode 100644 index 00000000..3afa1d98 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recommendation-5.test.ts @@ -0,0 +1,87 @@ +import { expect, test } from "bun:test"; +import { planEvents } from "./policy"; +import { seeded, seededOptionId, withOption } from "./interpret.test-fixtures"; +import { first, follow, message } from "./policy-initial.test-fixtures"; +import { optionChoice } from "./pipeline-ordinary-save.test-fixtures"; + +test("an existing-option recommendation suggests the option when Jev's role is uncertain", () => { + let state = withOption(seeded()); + for ( + let [id, text, roleProbability, supportProbability, triageResolution, targetingResolution] of [ + ["s3", "We should do amazon S3 for uploaded files", 0.68, 0.26, 0.68, 0.89], + ["search", "And we should use elastic search", 0.44, 0.39, 0.74, 0.87], + ] as const + ) { + let current = message(id, text); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: { + ...first({ explicit_resolution: triageResolution }), + act: { + type: "choice", + choice: "proposal", + confidence: 0.9, + probabilities: { proposal: 0.9, evaluation: 0.1 }, + }, + }, + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ + role: "resolution", + thread: "thread-a", + explicit_resolution: targetingResolution, + }), + role: { + type: "choice", + choice: "resolution", + confidence: roleProbability, + probabilities: { + resolution: roleProbability, + support: supportProbability, + none: 1 - roleProbability - supportProbability, + }, + }, + support: { type: "noul", noul: 0.89 }, + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(output.events.map(event => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ + optionId: seededOptionId, + source: { quote: text, messageId: id }, + }); + expect(state.threads[0].status).toBe("exploring"); + } +}); + +test("an apostrophe-free let's choice suggests Jev's exact existing option", () => { + let state = withOption(seeded()); + let current = message("postmark", "Lets do postmark for notifications"); + let output = planEvents({ + channelId: "channel", + message: current, + state, + first: first({ explicit_resolution: 0.92 }), + candidates: [{ + quote: current.text, + start: 0, + end: current.text.length, + answers: { + ...follow({ role: "resolution", thread: "thread-a", explicit_resolution: 0.94 }), + chosen_option: optionChoice(seededOptionId), + }, + }], + }); + expect(output.events.map(event => event.type)).toEqual(["settle.suggested"]); + expect(output.events[0]).toMatchObject({ + optionId: seededOptionId, + source: { quote: current.text, messageId: current.id }, + }); + expect(state.threads[0].status).toBe("exploring"); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recommendation-7.test.ts b/apps/server/src/conversation-plan/pipeline-recommendation-7.test.ts new file mode 100644 index 00000000..c3594043 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recommendation-7.test.ts @@ -0,0 +1,78 @@ +import { expect, test } from "bun:test"; +import { interpretMessage } from "./interpret"; +import { mockResult, seeded } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; + +test("multi-claim current text gets distinct source ranges and target choices", async () => { + let current = message( + "m5", + "Keep the outline optional; separately, should each repository remember my choice?", + ); + let seen: Record[] = []; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: seeded(), + ask: async (request) => { + seen.push(request.state as Record); + if (seen.length === 1) { + return mockResult(request.questions, { + new_question: 0.9, + support: 0.9, + significance: 2, + }); + } + return mockResult(request.questions, { + c0_role: "support", + c0_support: 0.95, + c0_thread: "thread-a", + c1_role: "question", + c1_thread: "new", + }); + }, + }); + expect(seen).toHaveLength(3); + expect(output.events.map((event) => event.type)).toContain("thread.opened"); + expect(output.events.map((event) => event.type)).toContain("stance.changed"); + expect("source" in output.events[0] ? output.events[0].source?.quote : undefined).not.toBe( + "source" in output.events[1] ? output.events[1].source?.quote : undefined, + ); +}); + +test("mixed message keeps per-candidate ranges and review gate in analysis", async () => { + let current = message( + "mixed", + "We've decided to use an outline. Should we remember the choice?", + ); + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: seeded(), + ask: async (request) => { + calls++; + return mockResult( + request.questions, + calls === 1 + ? { explicit_resolution: 0.98, new_question: 0.95 } + : { + c0_role: "resolution", + c0_thread: "none", + c0_explicit_resolution: 0.98, + c1_role: "question", + c1_thread: "new", + }, + ); + }, + }); + expect(output.events.map((event) => event.type)).toEqual(["thread.opened"]); + expect(output.analysis.outcomes).toHaveLength(2); + expect(output.analysis.outcomes?.[0]).toMatchObject({ status: "review", start: 0 }); + expect(output.analysis.outcomes?.[0].gate).toContain("target"); + expect(output.analysis.outcomes?.[1]).toMatchObject({ + status: "accepted", + eventIds: [output.events[0].id], + }); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recovery-alternatives.test.ts b/apps/server/src/conversation-plan/pipeline-recovery-alternatives.test.ts new file mode 100644 index 00000000..e4efb7d5 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recovery-alternatives.test.ts @@ -0,0 +1,91 @@ +import { expect, test } from "bun:test"; +import { seeded } from "./interpret.test-fixtures"; +import { interpretLowOwnership } from "./pipeline-recovery.test-fixtures"; + +test.each([ + { + id: "notification-email-providers", + text: "Which service should send notification emails: Postmark or Amazon SES?", + options: ["Postmark", "Amazon SES"], + }, + { + id: "authentication-approaches", + text: "Should we use Auth0, roll our own, or use GitHub auth?", + options: ["use Auth0", "roll our own", "use GitHub auth"], + }, + { + id: "audit-log-storage", + text: "Should audit logs go in PostgreSQL or object storage?", + options: ["PostgreSQL", "object storage"], + }, + { + id: "background-job-queues", + text: "Should background jobs use Redis or a PostgreSQL queue?", + options: ["Redis", "a PostgreSQL queue"], + }, +])("direct alternatives in $id retain exact spans despite low classifier ownership", async ({ + id, + text, + options, +}) => { + let output = await interpretLowOwnership(id, text); + expect(output.events.map(event => event.type)).toEqual([ + "thread.opened" as const, + ...options.map(() => "option.added" as const), + ]); + expect(output.events[0]).toMatchObject({ + question: text, + source: { quote: text, start: 0, end: text.length, role: "question" }, + }); + let optionEvents = output.events.slice(1); + expect(optionEvents.map(event => event.type === "option.added" ? event.contribution.text : "")) + .toEqual([...options]); + expect(optionEvents.map(event => + event.type === "option.added" + ? { + quote: event.source?.quote, + start: event.source?.start, + end: event.source?.end, + } + : undefined + )) + .toEqual(options.map(option => { + let start = text.indexOf(option); + return { quote: option, start, end: start + option.length }; + })); +}); + +test("a direct alternatives question opens beside an unrelated existing thread", async () => { + let text = "Should audit logs go in PostgreSQL or object storage?"; + let options = ["PostgreSQL", "object storage"]; + let output = await interpretLowOwnership("audit-with-unrelated-thread", text, { + weakFragments: false, + state: seeded(), + candidateThreadTarget: "none", + }); + expect(output.events.map(event => event.type)).toEqual([ + "thread.opened", + "option.added", + "option.added", + ]); + expect(output.events[0]).toMatchObject({ + question: text, + source: { quote: text, start: 0, end: text.length, role: "question" }, + }); + let optionEvents = output.events.slice(1); + expect(optionEvents.map(event => event.type === "option.added" ? event.contribution.text : "")) + .toEqual(options); + expect(optionEvents.map(event => + event.type === "option.added" + ? { + quote: event.source?.quote, + start: event.source?.start, + end: event.source?.end, + } + : undefined + )) + .toEqual(options.map(option => { + let start = text.indexOf(option); + return { quote: option, start, end: start + option.length }; + })); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recovery-ambiguity.test.ts b/apps/server/src/conversation-plan/pipeline-recovery-ambiguity.test.ts new file mode 100644 index 00000000..78856be5 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recovery-ambiguity.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { interpretLowOwnership, stateWithQuestion } from "./pipeline-recovery.test-fixtures"; + +test("ambiguous exact-question matches do not recover a partial card", async () => { + let text = "Should audit logs go in PostgreSQL or object storage?"; + let state = stateWithQuestion("thread:first-audit", text, ["PostgreSQL"]); + state = stateWithQuestion("thread:second-audit", text, [], state); + let output = await interpretLowOwnership("repeat-audit-ambiguous-state", text, { + weakFragments: false, + state, + triageTarget: "new", + candidateThreadTargets: { c0_thread: "new", c1_thread: "new" }, + candidateRoles: { c0_role: "question", c1_role: "option" }, + candidateDuplicates: { c0_duplicate: 0.95 }, + }); + expect(output.events).toEqual([]); +}); + +test.each([ + { + id: "reported-email-question", + text: "Mina asked, “Should we use Postmark or Amazon SES for notification emails?”", + }, + { + id: "reported-email-provider-question", + text: "Which service did Alice say we should use: Postmark or SES?", + }, + { + id: "negated-email-choice", + text: "Should we not use Postmark or Amazon SES for notification emails?", + }, + { + id: "negated-email-provider-question", + text: "Which service shouldn't we use: Postmark or SES?", + }, + { + id: "curly-negated-email-provider-question", + text: "Which service shouldn’t we use: Postmark or SES?", + }, + { + id: "indirect-email-question", + text: "Would it make sense to use Postmark or Amazon SES for notification emails?", + }, + { + id: "mixed-context-email-question", + text: "Quick question. Should we use Postmark or Amazon SES?", + }, +])("low ownership keeps $id from opening a sourced choice", async ({ id, text }) => { + let output = await interpretLowOwnership(id, text, { weakFragments: false }); + expect(output.events).toEqual([]); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recovery-refusal.test.ts b/apps/server/src/conversation-plan/pipeline-recovery-refusal.test.ts new file mode 100644 index 00000000..9919c7d7 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recovery-refusal.test.ts @@ -0,0 +1,51 @@ +import { expect, test } from "bun:test"; +import { initialState } from "./domain"; +import { interpretMessage } from "./interpret"; +import { mockResult } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; + +test.each( + [ + [ + "option-only prose", + "Auth0 is an option. Custom login is another.", + { new_question: 0.05, act: "proposal", thread_target: "none", new_option: 0.95 }, + { c0_role: "option", c0_thread: "none", c1_role: "option", c1_thread: "none" }, + ], + [ + "mixed candidate roles", + "Auth0 or custom login? React or Vue?", + { new_question: 0.97, act: "question", thread_target: "new" }, + { c0_role: "option", c0_thread: "new", c1_role: "reason", c1_thread: "none" }, + ], + [ + "uncertain new target", + "Should we use Auth0? Or custom login?", + { new_question: 0.97, act: "question", thread_target: "none" }, + { c0_role: "option", c0_thread: "new", c1_role: "option", c1_thread: "none" }, + ], + [ + "repeated option labels", + "Use Auth0. Use Auth0.", + { new_question: 0.97, act: "question", thread_target: "new" }, + { c0_role: "option", c0_thread: "new", c1_role: "option", c1_thread: "none" }, + ], + ] as const, +)("multi-option grouping refuses %s", async (_, text, triage, targeting) => { + let current = message("not-a-group", text); + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: initialState(), + ask: async request => + mockResult( + request.questions, + calls++ === 0 + ? { ...triage, significance: 2 } + : { ...targeting, c0_new_option: 0.95, c1_new_option: 0.95 }, + ), + }); + expect(output.events).toEqual([]); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recovery-repeat.test.ts b/apps/server/src/conversation-plan/pipeline-recovery-repeat.test.ts new file mode 100644 index 00000000..2a263640 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recovery-repeat.test.ts @@ -0,0 +1,123 @@ +import { expect, test } from "bun:test"; +import type { ConversationPlan } from "@chopin/protocol"; +import { applyInference, initialState } from "./domain"; +import { message } from "./policy-initial.test-fixtures"; +import { + interpretLowOwnership, + repeatQuestionCases, + stateWithQuestion, +} from "./pipeline-recovery.test-fixtures"; + +test.each([ + { id: "same-thread", targetKind: "same" }, + { id: "ambiguous-target", targetKind: "none" }, +])( + "a repeated audit question does not open another thread for $id", + async ({ id, targetKind }) => { + let text = "Should audit logs go in PostgreSQL or object storage?"; + let existing = message("existing-audit-question", text); + let threadId = "thread:existing-audit"; + let state = applyInference(initialState(), { + id: "existing-audit-thread", + type: "thread.opened", + threadId, + observedThreadVersion: 0, + origin: "classifier", + actor: { kind: "classifier" }, + at: existing.ts, + source: { + messageId: existing.id, + author: existing.author as ConversationPlan.SourceAuthor, + quote: text, + start: 0, + end: text.length, + role: "question", + }, + question: text, + }, existing); + let target = targetKind === "same" ? threadId : "none"; + let output = await interpretLowOwnership(`repeat-audit-${id}`, text, { + weakFragments: false, + state, + triageTarget: target, + candidateThreadTarget: target, + }); + expect(output.events.some(event => event.type === "thread.opened")).toBe(false); + }, +); + +test.each(repeatQuestionCases)( + "a repeated audit question repairs only missing options on an $id despite a new target", + async ({ id, existing, missing, duplicates }) => { + let text = "Should audit logs go in PostgreSQL or object storage?"; + let threadId = "thread:existing-audit"; + let state = stateWithQuestion(threadId, text, existing); + let output = await interpretLowOwnership(`repeat-audit-${id}`, text, { + weakFragments: false, + state, + triageTarget: "new", + candidateThreadTargets: { c0_thread: "new", c1_thread: "new" }, + candidateRoles: { c0_role: "question", c1_role: "option" }, + candidateDuplicates: duplicates, + }); + let options = output.events.flatMap(event => event.type === "option.added" ? [event] : []); + expect(output.analysis.passes[0]?.answers.thread_target).toMatchObject({ + type: "choice", + choice: "new", + }); + expect(output.events.some(event => event.type === "thread.opened")).toBe(false); + expect(options.map(event => event.contribution.text)).toEqual([...missing]); + expect(options.every(event => event.threadId === threadId)).toBe(true); + expect(options.map(event => ({ + quote: event.source?.quote, + start: event.source?.start, + end: event.source?.end, + }))).toEqual(missing.map(option => { + let start = text.indexOf(option); + return { quote: option, start, end: start + option.length }; + })); + }, +); + +test.each([ + { id: "matching-thread-target", target: "existing" }, + { id: "mistaken-new-target", target: "new" }, +])( + "a low new-question score still recovers an empty exact match with $id", + async ({ id, target }) => { + let text = "Should audit logs go in PostgreSQL or object storage?"; + let options = ["PostgreSQL", "object storage"]; + let threadId = "thread:existing-audit-low-score"; + let state = stateWithQuestion(threadId, text); + let targetId = target === "existing" ? threadId : "new"; + let output = await interpretLowOwnership(`repeat-audit-low-${id}`, text, { + state, + triageTarget: targetId, + newQuestion: 0.5, + candidateThreadTargets: { c0_thread: targetId, c1_thread: targetId }, + }); + let additions = output.events.flatMap(event => event.type === "option.added" ? [event] : []); + expect(output.analysis.passes[0]?.answers.new_question).toMatchObject({ + type: "noul", + noul: 0.5, + }); + expect(output.analysis.passes[0]?.answers.act).toMatchObject({ + type: "choice", + choice: "question", + }); + expect(output.events.map(event => event.type)).toEqual([ + "option.added", + "option.added", + ]); + expect(additions.map(event => event.contribution.text)).toEqual(options); + expect(additions.every(event => event.threadId === threadId)).toBe(true); + expect(additions.map(event => ({ + quote: event.source?.quote, + start: event.source?.start, + end: event.source?.end, + }))).toEqual(options.map(option => { + let start = text.indexOf(option); + return { quote: option, start, end: start + option.length }; + })); + }, +); diff --git a/apps/server/src/conversation-plan/pipeline-recovery-unique.test.ts b/apps/server/src/conversation-plan/pipeline-recovery-unique.test.ts new file mode 100644 index 00000000..c352f230 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recovery-unique.test.ts @@ -0,0 +1,76 @@ +import { expect, test } from "bun:test"; +import { initialState } from "./domain"; +import { interpretMessage } from "./interpret"; +import { mockResult, seeded } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; + +test("a new question keeps two unique quoted options when a third repeats one", async () => { + let current = message( + "repeat-option", + "Should we use Auth0? Or GitHub auth? Or GitHub auth?", + ); + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: initialState(), + ask: async request => + mockResult( + request.questions, + calls++ === 0 + ? { new_question: 0.97, act: "question", thread_target: "new", significance: 2 } + : { + c0_role: "option", + c0_thread: "new", + c0_new_option: 0.95, + c1_role: "option", + c1_thread: "none", + c1_new_option: 0.95, + c2_role: "option", + c2_thread: "none", + c2_new_option: 0.95, + }, + ), + }); + expect(output.events.map(event => event.type)).toEqual([ + "thread.opened", + "option.added", + "option.added", + ]); + expect( + output.events.flatMap(event => event.type === "option.added" ? [event.source?.quote] : []), + ).toEqual([ + "Should we use Auth0?", + "Or GitHub auth?", + ]); + expect(output.analysis.outcomes?.[2]?.gate).toBe("duplicate option in new question"); +}); + +test("multi-option grouping leaves options for an existing thread on that thread", async () => { + let state = seeded(); + let current = message("existing-options", "Use Auth0. Use custom login."); + let calls = 0; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state, + ask: async request => + mockResult( + request.questions, + calls++ === 0 + ? { new_question: 0.97, act: "question", thread_target: "new", significance: 2 } + : { + c0_role: "option", + c0_thread: "thread-a", + c0_new_option: 0.95, + c1_role: "option", + c1_thread: "thread-a", + c1_new_option: 0.95, + }, + ), + }); + expect(output.events.map(event => event.type)).toEqual(["option.added", "option.added"]); + expect(output.events.map(event => event.threadId)).toEqual(["thread-a", "thread-a"]); +}); diff --git a/apps/server/src/conversation-plan/pipeline-recovery.test-fixtures.ts b/apps/server/src/conversation-plan/pipeline-recovery.test-fixtures.ts new file mode 100644 index 00000000..1676f8ac --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-recovery.test-fixtures.ts @@ -0,0 +1,172 @@ +import type { ConversationPlan } from "@chopin/protocol"; +import { applyInference, initialState } from "./domain"; +import { interpretMessage } from "./interpret"; +import { mockResult } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; + +export let interpretLowOwnership = ( + id: string, + text: string, + options: { + weakFragments?: boolean; + state?: ConversationPlan.State; + triageTarget?: string; + newQuestion?: number; + candidateThreadTarget?: string; + candidateThreadTargets?: Record; + candidateRoles?: Record; + candidateDuplicates?: Record; + } = {}, +) => { + let { + weakFragments = true, + state = initialState(), + triageTarget = "new", + newQuestion = 0.97, + candidateThreadTarget, + candidateThreadTargets, + candidateRoles, + candidateDuplicates, + } = options; + return interpretMessage({ + channelId: "channel", + message: message(`low-ownership-${id}`, text, "Jules"), + recent: [], + state, + ask: async request => { + if ("new_question" in request.questions) { + let overrides: Record = { + new_question: newQuestion, + new_option: 0.36, + act: "question", + thread_target: triageTarget, + significance: 2, + }; + for (let key of Object.keys(request.questions)) { + if (key.endsWith("_owned_unretracted")) overrides[key] = 0.6; + } + return mockResult(request.questions, overrides); + } + let overrides: Record = {}; + for (let key of Object.keys(request.questions)) { + if (!key.startsWith("c")) continue; + if (key.endsWith("_role")) { + overrides[key] = candidateRoles?.[key] + ?? (weakFragments && key.startsWith("c0_") ? "none" : "option"); + } + if (key.endsWith("_thread")) { + overrides[key] = candidateThreadTargets?.[key] + ?? (weakFragments && key.startsWith("c0_") + ? "none" + : candidateThreadTarget ?? "new"); + } + if (key.endsWith("_new_option")) { + overrides[key] = weakFragments && key.startsWith("c0_") ? 0.7 : 0.95; + } + if (key.endsWith("_duplicate") && candidateDuplicates?.[key] !== undefined) { + overrides[key] = candidateDuplicates[key]!; + } + } + let result = mockResult(request.questions, overrides); + let firstRole = result.answers.c0_role; + let roleQuestion = request.questions.c0_role; + if ( + weakFragments && firstRole?.type === "choice" && roleQuestion?.type === "choice" + ) { + let probabilities = Object.fromEntries( + Object.keys(roleQuestion.criteria).map(key => [ + key, + key === "none" ? 0.55 : key === "option" ? 0.45 : 0, + ]), + ); + result.answers.c0_role = { + type: "choice", + choice: "none", + confidence: 0.45, + probabilities, + }; + } + return result; + }, + }); +}; + +export let stateWithQuestion = ( + threadId: string, + text: string, + options: readonly string[] = [], + state = initialState(), +) => { + let original = message(`existing-${threadId}`, text); + state = applyInference(state, { + id: `existing-${threadId}-question`, + type: "thread.opened", + threadId, + observedThreadVersion: 0, + origin: "classifier", + actor: { kind: "classifier" }, + at: original.ts, + source: { + messageId: original.id, + author: original.author as ConversationPlan.SourceAuthor, + quote: text, + start: 0, + end: text.length, + role: "question", + }, + question: text, + }, original); + for (let [index, option] of options.entries()) { + let start = text.indexOf(option); + state = applyInference(state, { + id: `existing-${threadId}-option-${index}`, + type: "option.added", + threadId, + observedThreadVersion: state.threads.find(thread => thread.id === threadId)!.version, + origin: "classifier", + actor: { kind: "classifier" }, + at: original.ts, + source: { + messageId: original.id, + author: original.author as ConversationPlan.SourceAuthor, + quote: option, + start, + end: start + option.length, + role: "option", + }, + contribution: { + id: `existing-${threadId}-option-${index}`, + text: option, + authoring: "quoted", + targetId: threadId, + }, + }, original); + } + return state; +}; + +export let repeatQuestionCases: Array<{ + id: string; + existing: string[]; + missing: string[]; + duplicates: Record; +}> = [ + { + id: "empty-card", + existing: [], + missing: ["PostgreSQL", "object storage"], + duplicates: {}, + }, + { + id: "partial-card", + existing: ["PostgreSQL"], + missing: ["object storage"], + duplicates: { c0_duplicate: 0.95 }, + }, + { + id: "complete-card", + existing: ["PostgreSQL", "object storage"], + missing: [], + duplicates: { c0_duplicate: 0.95, c1_duplicate: 0.95 }, + }, +]; diff --git a/apps/server/src/conversation-plan/pipeline-reply-options.test.ts b/apps/server/src/conversation-plan/pipeline-reply-options.test.ts new file mode 100644 index 00000000..720b31c9 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-reply-options.test.ts @@ -0,0 +1,107 @@ +import { expect, test } from "bun:test"; +import { initialState } from "./domain"; +import { interpretMessage } from "./interpret"; +import { mockResult } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; + +test.each([ + { + id: "uploaded-files", + text: "Should we store uploaded files in Amazon S3 or on a local disk?", + options: ["Amazon S3", "on a local disk"], + }, + { + id: "notification-emails", + text: "Which service should send notification emails: Postmark or Amazon SES?", + options: ["Postmark", "Amazon SES"], + }, + { + id: "d03-notification-providers", + text: "what sends transactional notifications? our SMTP relay, Postmark, or SES?", + options: ["SMTP relay", "Postmark", "SES"], + }, +])("extracts only the exact alternatives from $id question", async ({ id, text, options }) => { + let current = message(`explicit-alternatives-${id}`, text, "Jules"); + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: initialState(), + ask: async request => { + if ("new_question" in request.questions) { + return mockResult(request.questions, { + new_question: 0.97, + act: "question", + thread_target: "new", + significance: 2, + }); + } + let overrides: Record = {}; + for (let key of Object.keys(request.questions)) { + if (!key.startsWith("c")) continue; + if (key.endsWith("_role")) overrides[key] = "option"; + if (key.endsWith("_thread")) overrides[key] = "new"; + if (/^c\d+_option$/.test(key)) overrides[key] = "new"; + if (key.endsWith("_new_option")) overrides[key] = 0.95; + } + return mockResult(request.questions, overrides); + }, + }); + + expect(output.events.map(event => event.type)).toEqual([ + "thread.opened", + ...options.map(() => "option.added" as const), + ]); + expect(output.events[0]).toMatchObject({ + question: text, + source: { quote: text, start: 0, end: text.length, role: "question" }, + }); + let optionEvents = output.events.slice(1); + expect(optionEvents.map(event => event.type === "option.added" ? event.contribution.text : "")) + .toEqual([...options]); + expect(optionEvents.map(event => + event.type === "option.added" + ? { + quote: event.source?.quote, + start: event.source?.start, + end: event.source?.end, + } + : undefined + )) + .toEqual(options.map(option => { + let start = text.indexOf(option); + return { quote: option, start, end: start + option.length }; + })); +}); + +test("D03's direct option group requires option=new for every candidate", async () => { + let text = "what sends transactional notifications? our SMTP relay, Postmark, or SES?"; + let current = message("d03-missing-option-target", text, "Jules"); + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent: [], + state: initialState(), + ask: async request => { + if ("new_question" in request.questions) { + return mockResult(request.questions, { + new_question: 0.97, + act: "question", + thread_target: "new", + significance: 2, + }); + } + let overrides: Record = {}; + for (let key of Object.keys(request.questions)) { + if (!key.startsWith("c")) continue; + if (key.endsWith("_role")) overrides[key] = "option"; + if (key.endsWith("_thread")) overrides[key] = "new"; + let optionTarget = key.match(/^c(\d+)_option$/); + if (optionTarget) overrides[key] = optionTarget[1] === "1" ? "none" : "new"; + if (key.endsWith("_new_option")) overrides[key] = 0.95; + } + return mockResult(request.questions, overrides); + }, + }); + expect(output.events.filter(event => event.type === "option.added")).toEqual([]); +}); diff --git a/apps/server/src/conversation-plan/pipeline-source-fallback.test.ts b/apps/server/src/conversation-plan/pipeline-source-fallback.test.ts new file mode 100644 index 00000000..acfb7dd4 --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-source-fallback.test.ts @@ -0,0 +1,186 @@ +import { expect, test } from "bun:test"; +import type { ConversationPlan } from "@chopin/protocol"; +import type { JevAnswer } from "./jev"; +import { applyInference } from "./domain"; +import { applyEvent } from "./events"; +import { optionIdFor, planEvents } from "./policy"; +import { buildCandidateTargetingRequest } from "./questions"; +import { extractQuotes } from "./quotes"; +import { confidentChoice, first, follow, member, message } from "./policy-initial.test-fixtures"; +import { addThreadWithOption } from "./policy-terminal.test-fixtures"; +import { hostingState } from "./pipeline-commitment.test-fixtures"; + +test("a sourced fallback condition qualifies the pending VPS choice only with a clear referent", () => { + let state = hostingState(); + let managedOptionId = optionIdFor("channel", "managed-hosting-option", 0, 1000); + state = applyEvent(state, { + id: "managed-hosting-option-event", + type: "option.added", + threadId: "hosting-thread", + observedThreadVersion: state.threads[0]!.version, + origin: "human", + actor: member("Mina"), + at: 1000, + contribution: { + id: managedOptionId, + text: "Use managed hosting.", + authoring: "human-edited", + }, + }); + state = addThreadWithOption( + state, + "other-thread", + "Where should backups live?", + "backup-option", + "Keep backups on object storage.", + ); + let proposal = message("vps-proposal", "Let's host it on our own VPS.", "Mina"); + state = applyInference(state, { + id: "vps-pending-event", + type: "settle.suggested", + threadId: "hosting-thread", + observedThreadVersion: state.threads[0]!.version, + origin: "classifier", + actor: { kind: "classifier" }, + at: proposal.ts, + source: { + messageId: proposal.id, + author: proposal.author as ConversationPlan.SourceAuthor, + quote: proposal.text, + start: 0, + end: proposal.text.length, + role: "resolution", + }, + optionId: "vps-option", + }, proposal); + let current = message( + "qualified-vps", + "yes, and keep managed hosting as the escape hatch if on-call gets silly", + "Jules", + ); + let quotes = extractQuotes(current.text); + let isolated = buildCandidateTargetingRequest( + current, + [proposal], + state.threads, + quotes, + 1, + state.events, + ); + expect(isolated.questions.c1_qualifies_pending_settle?.type).toBe("noul"); + let condition: Parameters[0]["candidates"][number] = { + ...quotes[1]!, + answers: { + ...follow({ role: "constraint", thread: "hosting-thread" }), + role: { + type: "choice" as const, + choice: "constraint", + confidence: 0.46, + probabilities: { constraint: 0.46, support: 0.3, option: 0.14, none: 0.1 }, + }, + option: confidentChoice("none"), + relation: { + type: "choice" as const, + choice: "qualifies", + confidence: 0.76, + probabilities: { qualifies: 0.76, supports: 0.18, unrelated: 0.06 }, + }, + planning_substance: { type: "noul" as const, noul: 0.91 }, + qualifies_pending_settle: { type: "noul" as const, noul: 0.91 }, + duplicate: { type: "noul" as const, noul: 0.08 }, + }, + }; + let triage: Record = { + ...first({ constraint: 0.9 }), + significance: { + type: "score", + score: 2, + confidence: 1, + legend: { "0": "none", "1": "minor", "2": "useful" }, + probabilities: { "0": 0, "1": 0, "2": 1 }, + }, + }; + let assess = ( + candidate = condition, + firstAnswers = triage, + candidateState = state, + ) => + planEvents({ + channelId: "channel", + message: current, + state: candidateState, + first: firstAnswers, + candidates: [candidate], + }); + let accepted = assess(); + expect(accepted.events.map(event => event.type)).toEqual(["constraint.added"]); + expect(accepted.events[0]).toMatchObject({ + threadId: "hosting-thread", + source: { ...quotes[1], role: "constraint" }, + contribution: { targetId: "vps-option", relation: "qualifies" }, + }); + let strongFallbackSupport = assess({ + ...condition, + answers: { + ...condition.answers, + role: confidentChoice("support"), + support: { type: "noul" as const, noul: 0.96 }, + option: confidentChoice(managedOptionId), + }, + }, { ...triage, support: { type: "noul", noul: 0.96 } }); + expect(strongFallbackSupport.events.map(event => event.type)).toEqual(["constraint.added"]); + expect(strongFallbackSupport.events[0]).toMatchObject({ + contribution: { targetId: "vps-option", relation: "qualifies" }, + }); + expect( + assess({ + ...condition, + answers: { + ...condition.answers, + option: confidentChoice("backup-option"), + }, + }).events[0], + ).toMatchObject({ contribution: { targetId: "vps-option" } }); + let guards: Record[] = [ + { thread: confidentChoice("none") }, + { relation: confidentChoice("unrelated") }, + { qualifies_pending_settle: { type: "noul" as const, noul: 0.4 } }, + { planning_substance: { type: "noul" as const, noul: 0.4 } }, + { duplicate: { type: "noul" as const, noul: 0.9 } }, + ]; + for (let answers of guards) { + expect(assess({ ...condition, answers: { ...condition.answers, ...answers } }).events) + .toEqual([]); + } + expect( + assess(condition, { + ...triage, + c0_owned_unretracted: { type: "noul", noul: 0.2 }, + }).events, + ).toEqual([]); + let stale = structuredClone(state); + stale.threads[0]!.pendingSettle = undefined; + expect(assess(condition, triage, stale).events).toEqual([]); + let withdrawal = message("vps-withdrawal", "I no longer support the VPS.", "Mina"); + let withdrawn = applyInference(state, { + id: "vps-withdrawal-event", + type: "stance.changed", + scopedProposalId: null, + threadId: "hosting-thread", + observedThreadVersion: state.threads[0]!.version, + origin: "classifier", + actor: { kind: "classifier" }, + at: withdrawal.ts, + source: { + messageId: withdrawal.id, + author: withdrawal.author as ConversationPlan.SourceAuthor, + quote: withdrawal.text, + start: 0, + end: withdrawal.text.length, + role: "objection", + }, + optionId: "vps-option", + position: "oppose", + }, withdrawal); + expect(assess(condition, triage, withdrawn).events).toEqual([]); +}); diff --git a/apps/server/src/conversation-plan/pipeline-source-targeting.test.ts b/apps/server/src/conversation-plan/pipeline-source-targeting.test.ts new file mode 100644 index 00000000..ece4eebe --- /dev/null +++ b/apps/server/src/conversation-plan/pipeline-source-targeting.test.ts @@ -0,0 +1,111 @@ +import { expect, test } from "bun:test"; +import type { JevQuestion } from "./jev"; +import { interpretMessage } from "./interpret"; +import { buildTargetingRequest } from "./questions"; +import { extractQuotes } from "./quotes"; +import { mockResult, seeded, settledBy, withOption } from "./interpret.test-fixtures"; +import { message } from "./policy-initial.test-fixtures"; + +test("active targets after the eighth keep their options in choices and compact context", () => { + let template = withOption(seeded()).threads[0]; + let discarded = { ...template, id: "discarded-first", status: "discarded" as const }; + let active = Array.from({ length: 12 }, (_, index) => ({ + ...template, + id: `thread-${index}`, + question: `Question ${index}?`, + contributions: [{ ...template.contributions[0], id: `option-${index}` }], + })); + let current = message("late-target", "Let's settle option eleven."); + let request = buildTargetingRequest( + current, + [], + [discarded, ...active], + extractQuotes(current.text), + ); + let threadChoices = + (request.questions.c0_thread as Extract).criteria; + let optionChoices = + (request.questions.c0_chosen_option as Extract).criteria; + expect(threadChoices["thread-11"]).toContain("Question 11"); + expect(threadChoices["discarded-first"]).toBeUndefined(); + expect(optionChoices["option-11"]).toContain("Question 11"); + let context = + (request.state as { threads: Array<{ id: string; options: Array<{ id: string }> }> }) + .threads; + expect(context.map((thread) => thread.id)).toHaveLength(12); + expect(context.find((thread) => thread.id === "thread-11")?.options.map((option) => option.id)) + .toContain("option-11"); +}); + +test("four isolated targets persist a pending qualifier relation within 45 answers", async () => { + let current = message( + "figma-mixed", + "Sounds good to me. What about optional outlines if editors can change them? Copilot? BYO API keys?", + "Jules", + ); + let recent = [ + message("purpose", "We need to figure out auth.", "Mina"), + message("alternatives", "Auth0, custom login, or GitHub auth?", "Jules"), + message("suggestion", "GitHub could work.", "Mina"), + message("proposal", "Let's just go with GitHub.", "Mina"), + ]; + let calls: Array<{ questions: string[]; state: unknown }> = []; + let output = await interpretMessage({ + channelId: "channel", + message: current, + recent, + state: settledBy(seeded(), "Mina"), + ask: async request => { + let keys = Object.keys(request.questions); + calls.push({ questions: keys, state: request.state }); + let overrides: Record = keys.includes("new_question") + ? { + new_question: 0.95, + support: 0.9, + significance: 2, + c0_owned_unretracted: 0.95, + c1_owned_unretracted: 0.95, + c2_owned_unretracted: 0.95, + } + : keys.some(key => key.startsWith("c0_")) + ? { + c0_role: "support", + c0_thread: "thread-a", + c0_support: 0.95, + c0_agrees_with_settle: 0.95, + } + : keys.some(key => key.startsWith("c1_")) + ? { c1_role: "question", c1_thread: "new" } + : {}; + return mockResult(request.questions, overrides); + }, + }); + expect(calls).toHaveLength(5); + expect(calls[0]!.questions).toContain("new_question"); + for (let index = 0; index < 4; index++) { + expect(calls[index + 1]!.questions.every(key => key.startsWith(`c${index}_`))) + .toBe(true); + expect((calls[index + 1]!.state as { recent: Array<{ id: string }> }).recent + .map(entry => entry.id)).toEqual(recent.map(entry => entry.id)); + } + expect(calls[2]!.questions).toContain("c1_relation"); + expect(calls[2]!.questions).toContain("c1_qualifies_pending_settle"); + expect(output.analysis.passes.map(pass => pass.stage)).toEqual(["triage", "targeting"]); + expect(calls.slice(1).flatMap(call => call.questions).filter(key => key.endsWith("_duplicate"))) + .toEqual([]); + expect(Object.keys(output.analysis.passes[1]!.answers)).toHaveLength(44); + expect(output.events.map(event => event.type)).toEqual([ + "stance.changed", + "settle.agreed", + "thread.opened", + ]); + let quotes = extractQuotes(current.text); + expect(output.events.map(event => "source" in event ? event.source?.quote : "")) + .toEqual([quotes[0]!.quote, quotes[0]!.quote, quotes[1]!.quote]); + expect(output.events.every(event => + "source" in event + && event.source?.messageId === current.id + && current.text.slice(event.source.start, event.source.end) === event.source.quote + )) + .toBe(true); +}); diff --git a/apps/server/src/conversation-plan/question-interpreter.test.ts b/apps/server/src/conversation-plan/question-interpreter.test.ts new file mode 100644 index 00000000..b82838de --- /dev/null +++ b/apps/server/src/conversation-plan/question-interpreter.test.ts @@ -0,0 +1,118 @@ +import { expect, test } from "bun:test"; +import type { Chat, ConversationPlan } from "@chopin/protocol"; +import { initialState } from "./domain"; +import { interpretMessage } from "./interpret"; +import type { JevAnswer, JevRequest } from "./jev"; +import { cacheMessage, cacheQuotes, emptyCacheThread } from "./question-builder.test-fixtures"; + +// Whole final outer loop from archive446a9779a937fa5be7cd3eb52fd7f3023d691ed2, questions.test.ts. +for (let hasPrior of [false, true]) { + test(`interpretation ${hasPrior ? "accepts true" : "omits unasked"} duplicate evidence`, async () => { + let requests: JevRequest[] = []; + let earlier: Chat.Entry = { + ...cacheMessage, + id: "cache-earlier-option", + text: cacheQuotes[0]!.quote, + ts: cacheMessage.ts - 1, + }; + let thread: ConversationPlan.Thread = hasPrior + ? { + ...emptyCacheThread, + contributions: [{ + id: "cache-redis-option", + kind: "option", + text: cacheQuotes[0]!.quote, + authoring: "quoted", + sources: [{ + messageId: earlier.id, + author: { kind: "member", handle: "Ari" }, + quote: earlier.text, + start: 0, + end: earlier.text.length, + role: "option", + }], + actor: { kind: "classifier" }, + }], + } + : emptyCacheThread; + let interpreted = await interpretMessage({ + channelId: "engineering-cache", + message: cacheMessage, + recent: hasPrior ? [earlier] : [], + state: { ...initialState(), threads: [thread] }, + ask: async request => { + requests.push(request); + let answers = Object.fromEntries( + Object.entries(request.questions).map(([key, question]) => { + let answer: JevAnswer; + if (question.type === "noul") { + answer = { + type: "noul", + noul: key === "new_option" || key.endsWith("_owned_unretracted") + || hasPrior && key === "c0_duplicate" + ? 0.95 + : 0.05, + }; + } else if (question.type === "score") { + answer = { + type: "score", + score: 2, + confidence: 1, + legend: Object.fromEntries(question.criteria.map((label, index) => [index, label])), + probabilities: { "0": 0, "1": 0, "2": 1, "3": 0 }, + }; + } else { + let preferred = key === "act" + ? "proposal" + : key === "thread_target" + ? "api-cache" + : key.endsWith("_role") + ? "option" + : key.endsWith("_thread") + ? "api-cache" + : key.endsWith("_option") + ? "new" + : "none"; + let selected = preferred in question.criteria + ? preferred + : Object.keys(question.criteria)[0]!; + answer = { + type: "choice", + choice: selected, + confidence: 1, + probabilities: Object.fromEntries( + Object.keys(question.criteria).map(value => [ + value, + Number(value === selected), + ]), + ), + }; + } + return [key, answer]; + }), + ); + return { + model: "fake-jev", + answers, + usage: { input_tokens: 0, output_tokens: 0 }, + latencyMs: 0, + }; + }, + }); + let targeting = requests.filter(request => !("new_question" in request.questions)); + expect(targeting).toHaveLength(2); + let pass = interpreted.analysis.passes.find(item => item.stage === "targeting"); + if (hasPrior) { + expect(targeting[0]?.questions.c0_duplicate?.type).toBe("noul"); + expect(pass?.answers.c0_duplicate).toEqual({ type: "noul", noul: 0.95 }); + } else { + expect( + targeting.every(request => + Object.keys(request.questions).every(key => !key.endsWith("_duplicate")) + ), + ).toBe(true); + expect(pass?.answers.c0_duplicate).toBeUndefined(); + expect(pass?.answers.c1_duplicate).toBeUndefined(); + } + }); +} diff --git a/apps/server/src/conversation-plan/recent-thread-context.test.ts b/apps/server/src/conversation-plan/recent-thread-context.test.ts new file mode 100644 index 00000000..dfc63167 --- /dev/null +++ b/apps/server/src/conversation-plan/recent-thread-context.test.ts @@ -0,0 +1,168 @@ +import { expect, test } from "bun:test"; +import type { ConversationPlan } from "@chopin/protocol"; +import { initialState } from "./domain"; +import { applyEvent } from "./events"; +import { + buildCandidateTargetingRequest, + buildResearchOfferRequest, + buildTargetingRequest, + buildTriageRequest, +} from "./questions"; +import { visibleThreads } from "./question-context"; +import { message } from "./policy-initial.test-fixtures"; +import { extractQuotes } from "./quotes"; +import { interpretMessage } from "./interpret"; +import { mockResult } from "./interpret.test-fixtures"; +import type { JevRequest } from "./jev"; + +function populated(count = 20): ConversationPlan.State { + let state = initialState(); + for (let index = 0; index < count; index++) { + let threadId = `thread-${index}`; + let quote = `Which provider for service ${index}?`; + let base = { + threadId, + origin: "classifier" as const, + actor: { kind: "classifier" } as const, + at: index + 1, + }; + let source = { + messageId: `message-${index}`, + author: { kind: "member", handle: "alice" } as const, + quote, + start: 0, + end: quote.length, + role: "question" as const, + }; + state = applyEvent(state, { + ...base, + id: `opened-${index}`, + type: "thread.opened", + observedThreadVersion: 0, + question: quote, + source, + }); + for (let option = 0; option < 2; option++) { + let text = `Provider ${index}-${option}`; + state = applyEvent(state, { + ...base, + id: `added-${index}-${option}`, + type: "option.added", + observedThreadVersion: option + 1, + contribution: { id: `option-${index}-${option}`, text, authoring: "quoted" }, + source: { ...source, quote: text, end: text.length, role: "option" }, + }); + } + } + return state; +} + +function ids(selected: ReturnType): string[] { + return selected.map(({ thread }) => thread.id); +} + +test("a twentieth thread enters the bounded context while small contexts retain their order", () => { + let state = populated(); + let expected = state.threads.slice(-12).map(thread => thread.id); + expect(ids(visibleThreads(state.threads))).toEqual(expected); + expect(ids(visibleThreads(state.threads, state.events))).toEqual(expected); + let small = state.threads.slice(0, 12); + expect(ids(visibleThreads(small, state.events))).toEqual(small.map(thread => thread.id)); +}); + +test("accepted corrections refresh old threads by event order across every request builder", () => { + let state = populated(); + state = applyEvent(state, { + id: "latest-correction", + type: "card.corrected", + threadId: "thread-0", + observedThreadVersion: 3, + origin: "human", + actor: { kind: "member", handle: "alice" }, + at: 0, + change: { kind: "edit", field: "question", text: "Which current provider for service 0?" }, + }); + let expected = ["thread-0", ...state.threads.slice(-11).map(thread => thread.id)]; + let current = message("current", "We need current provider costs."); + let candidates = extractQuotes(current.text); + let requests = [ + buildTriageRequest(current, [], state.threads, candidates, state.events), + buildTargetingRequest(current, [], state.threads, candidates, undefined, state.events), + buildCandidateTargetingRequest(current, [], state.threads, candidates, 0, state.events), + buildResearchOfferRequest(current, [], state.threads, candidates, state.events), + ]; + for (let request of requests) { + let context = request.state as { threads: Array<{ id: string; options: unknown[] }> }; + expect(context.threads.map(thread => thread.id)).toEqual(expected); + expect(context.threads).toHaveLength(12); + expect(context.threads.flatMap(thread => thread.options).length).toBeLessThanOrEqual(32); + expect(JSON.stringify(request.state).length).toBeLessThanOrEqual(23_500); + } +}); + +test("a move refreshes both affected threads without promoting discarded threads over live ones", () => { + let state = populated(); + state = applyEvent(state, { + id: "move", + type: "card.corrected", + threadId: "thread-0", + observedThreadVersion: 3, + origin: "human", + actor: { kind: "member", handle: "alice" }, + at: 21, + change: { + kind: "move", + contributionId: "option-0-0", + targetThreadId: "thread-1", + targetVersion: 3, + }, + }); + expect(ids(visibleThreads(state.threads, state.events))).toEqual([ + "thread-0", + "thread-1", + ...state.threads.slice(-10).map(thread => thread.id), + ]); + state = applyEvent(state, { + id: "discard", + type: "thread.discarded", + threadId: "thread-19", + observedThreadVersion: 3, + origin: "human", + actor: { kind: "member", handle: "alice" }, + at: 22, + }); + expect(ids(visibleThreads(state.threads, state.events))).not.toContain("thread-19"); +}); + +test("production interpretation supplies the accepted recency to research and targeting", async () => { + let state = populated(); + state = applyEvent(state, { + id: "newest-touch", + type: "card.corrected", + threadId: "thread-0", + observedThreadVersion: 3, + origin: "human", + actor: { kind: "member", handle: "alice" }, + at: 0, + change: { kind: "edit", field: "question", text: "Which current provider for service 0?" }, + }); + let requests: JevRequest[] = []; + await interpretMessage({ + channelId: "channel", + message: message("current", "We need current provider costs."), + recent: [], + state, + ask: async request => { + requests.push(request); + return mockResult(request.questions, { research_need: 0.95, reason: 0.8 }); + }, + }); + expect(requests).toHaveLength(3); + for (let request of requests) { + let context = request.state as { threads: Array<{ id: string }> }; + expect(context.threads.map(thread => thread.id)).toEqual([ + "thread-0", + ...state.threads.slice(-11).map(thread => thread.id), + ]); + } +}); diff --git a/apps/server/src/conversation-plan/research-offers.test-fixtures.ts b/apps/server/src/conversation-plan/research-offers.test-fixtures.ts new file mode 100644 index 00000000..f619a8c6 --- /dev/null +++ b/apps/server/src/conversation-plan/research-offers.test-fixtures.ts @@ -0,0 +1,185 @@ +import type { Chat, ConversationPlan } from "@chopin/protocol"; +import { initialState, offerResearch } from "./domain"; +import { renderResearchTask } from "./validation"; +import { applyEvent } from "./events"; + +export function message(id: string, text = "Look up the 🧵 hosting terms"): Chat.Entry { + return { id, author: { kind: "member", handle: "ada" }, text, ts: 1 }; +} + +export function proposal( + entry: Chat.Entry, + id = `offer:${entry.id}`, + needId = "hosting-terms", + contextId = "v1", +): Omit { + return { + id, + needId, + contextId, + brief: entry.text, + source: { + messageId: entry.id, + author: entry.author as ConversationPlan.SourceAuthor, + quote: entry.text, + start: 0, + end: entry.text.length, + }, + }; +} + +export let ada: Extract = { kind: "member", handle: "ada" }; +export let bo: Extract = { kind: "member", handle: "bo" }; +export let s3Id = "01K00000000000000000000001"; +export let r2Id = "01K00000000000000000000002"; +export let diskId = "01K00000000000000000000003"; +export let postmarkId = "01K00000000000000000000004"; +export let sesId = "01K00000000000000000000005"; +export let mailgunId = "01K00000000000000000000006"; + +export type CostConcernTask = { + kind: "current-cost-concern"; + threadId: string; + observedEventCount: number; + observedThreadVersion: number; + options: Array<{ id: string; labelAtOffer: string }>; + focusOptionId?: string; +}; + +export type CostConcernProposal = + & Omit + & { task: CostConcernTask }; + +export function optionThread(): ConversationPlan.State { + let state = initialState(); + state = applyEvent(state, { + id: "open", + type: "thread.opened", + threadId: "hosting", + observedThreadVersion: 0, + origin: "planner", + actor: { kind: "agent" }, + at: 1, + question: "Which storage?", + }); + for (let [id, label] of [[s3Id, "S3"], [r2Id, "R2"]]) { + state = applyEvent(state, { + id: `add-${id}`, + type: "option.added", + threadId: "hosting", + observedThreadVersion: state.threads[0]!.version, + origin: "planner", + actor: { kind: "agent" }, + at: state.revision + 1, + contribution: { id, text: label, authoring: "scribe" }, + }); + } + return state; +} + +export function taskProposal( + entry: Chat.Entry, + state: ConversationPlan.State, +): Omit { + let task: ConversationPlan.ResearchTask = { + kind: "current-cost-comparison", + threadId: "hosting", + observedEventCount: state.events.length, + observedThreadVersion: state.threads[0]!.version, + options: [{ id: s3Id, labelAtOffer: "S3" }, { id: r2Id, labelAtOffer: "R2" }], + }; + return { + ...proposal(entry), + threadId: "hosting", + task, + brief: renderResearchTask(task, proposal(entry).source), + }; +} + +export function stateWithOptions( + threadId: string, + question: string, + options: Array<{ id: string; label: string }>, +): ConversationPlan.State { + let state = applyEvent(initialState(), { + id: `open-${threadId}`, + type: "thread.opened", + threadId, + observedThreadVersion: 0, + origin: "planner", + actor: { kind: "agent" }, + at: 1, + question, + }); + for (let option of options) { + state = applyEvent(state, { + id: `add-${option.id}`, + type: "option.added", + threadId, + observedThreadVersion: state.threads[0]!.version, + origin: "planner", + actor: { kind: "agent" }, + at: state.revision + 1, + contribution: { id: option.id, text: option.label, authoring: "scribe" }, + }); + } + return state; +} + +export function exactSource( + entry: Chat.Entry, + quote = entry.text, +): ConversationPlan.ResearchSource { + let start = entry.text.indexOf(quote); + if (start < 0) throw new Error("quote is missing from message"); + return { + messageId: entry.id, + author: entry.author as ConversationPlan.SourceAuthor, + quote, + start, + end: start + quote.length, + }; +} + +export function costConcernProposal( + entry: Chat.Entry, + state: ConversationPlan.State, + focusOptionId?: string, + quote = entry.text, + threadId = "storage", +): CostConcernProposal { + let thread = state.threads.find(item => item.id === threadId)!; + let source = exactSource(entry, quote); + let task: CostConcernTask = { + kind: "current-cost-concern", + threadId: thread.id, + observedEventCount: state.events.length, + observedThreadVersion: thread.version, + options: thread.contributions + .filter(item => item.kind === "option") + .map(item => ({ id: item.id, labelAtOffer: item.displayLabel ?? item.text })), + ...(focusOptionId ? { focusOptionId } : {}), + }; + return { + ...proposal(entry, `offer:${entry.id}`, "current-cost-concern", "storage-v1"), + source, + threadId: thread.id, + task, + brief: renderResearchTask( + task as unknown as ConversationPlan.ResearchTask, + source, + ), + }; +} + +export function offerCostConcern( + state: ConversationPlan.State, + proposal: CostConcernProposal, + entry: Chat.Entry, +) { + return offerResearch( + state, + proposal as unknown as Omit, + entry, + ); +}