diff --git a/.gitignore b/.gitignore index 763a5fa..5cab977 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,10 @@ dist/ *.tgz artifacts/ coverage/ + +# ── Bounded-autonomy durable state (per-workspace, never committed) ── +# Agent runtime state: phase locks, memory receipts, L4 phase/release receipts, +# and PR handoffs, written under L9_AUTONOMY_STATE_DIR (default .l9/). It is +# machine- and session-local — it carries session IDs and absolute paths, and +# is not source. Matches the convention in the rest of the constellation. +.l9/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b61298f..5042a29 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -19,6 +19,26 @@ validated execution task Route resolution is pure. Request IDs and timestamps are added afterward and do not participate in routing equivalence. +## Search policy authority + +The application declares the capability; the router selects the provider. That split is enforced by a single resolver, `resolveSearchPolicy()` in `src/matrices/search-policy.ts`: + +```text +typeof task.requiresSearch === 'boolean' + -> { required: task.requiresSearch, source: EXPLICIT } +otherwise + -> { required: isSearchTask(task.type), source: TASK_DEFAULT } +``` + +There is exactly one implementation of this rule. `requiresSearchProvider()` is a boolean view of it and `isSearchTask()` supplies only the `TaskType` default. `resolveRoute()` consumes the resolution and copies `searchRequired` and `searchPolicySource` onto every `RoutingResolution`, so a decision is auditable without inferring intent from model names. + +Two invariants keep the audit honest: + +- A resolved Perplexity config always has `disableSearch: false`. A search decision can never dispatch a config with web search turned off. +- Before dispatch, `decision.searchRequired` must equal `decision.provider === Provider.PERPLEXITY`. A disagreement in either direction is a hard error, not a downgrade. + +Search and vision have no combined provider contract. A vision `TaskType` carrying images with `requiresSearch: true` throws `UnsupportedCapabilityCombinationError` from route resolution — before request identity, budget reservation, circuit permit, or provider dispatch — so neither capability is silently discarded. Because the throw precedes reservation and permit acquisition, it cannot affect budget state or provider circuit health. + ## Module ownership ```text diff --git a/README.md b/README.md index 86852ad..602814b 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,53 @@ const router = new L9LLMRouter({ Resolution precedence is explicit config, then `OPENROUTER_BASE_URL`, then the OpenRouter cloud default. Overrides are validated as absolute http(s) URLs at construction time and trailing slashes are normalized. Invalid values throw `InvalidBaseUrlError` (or `RouterConfigValidationError` at config parse time). Deployments that set neither are unaffected. +## Search policy + +An application declares *what capability the task needs*. The router decides *which provider and model* serve it. `TaskDescriptor.requiresSearch` is the capability declaration, and when it is present it is authoritative: + +```ts +shouldSearch(task) = + typeof task.requiresSearch === 'boolean' + ? task.requiresSearch + : isSearchTask(task.type); +``` + +| `requiresSearch` | Result | `searchPolicySource` | +| --- | --- | --- | +| `true` | Search plane (Perplexity Sonar) | `EXPLICIT` | +| `false` | General plane, even for a research `TaskType` | `EXPLICIT` | +| omitted | The historical `TaskType` default | `TASK_DEFAULT` | + +The `TaskType` default is unchanged: `COMPETITOR_RESEARCH`, `CITATION_CHECK`, `FACT_VERIFICATION`, `MARKET_RESEARCH`, and `LINK_PROSPECTING` still route to search when the flag is omitted. Explicit `false` lets a caller reason strategically over evidence a deterministic system already gathered without paying for redundant web search; explicit `true` lets an otherwise-general task reach fresh web context. + +```ts +// Strategic synthesis over evidence we already hold — no web search. +await router.execute( + { clientId: 'tenant-a', type: TaskType.COMPETITOR_RESEARCH, complexity: TaskComplexity.HIGH, requiresSearch: false }, + 'You are a strategist.', + 'Synthesize the supplied competitor evidence.', +); +``` + +`TaskDescriptor` carries no `provider`, `model`, or fallback-chain field, and unknown keys are stripped during validation. Applications cannot select a provider or model. + +### Auditing a routing decision + +Every `RoutingDecision` — from `route()` and from `getCallLog()` — reports whether search was selected and on whose authority, alongside `taskType`, `complexity`, `provider`, `model`, `reason`, `estimatedCost`, `taskId`, `clientId`, `timestamp`, downgrade state, and (after execution) `actualCost` and `latencyMs`. No credentials or prompts are recorded. + +```ts +const decision = router.route({ clientId: 'tenant-a', type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.HIGH, requiresSearch: false }); +decision.searchRequired; // false +decision.searchPolicySource; // SearchPolicySource.EXPLICIT +decision.provider; // Provider.OPENROUTER +``` + +`searchRequired` always agrees with the plane actually dispatched; the router asserts this in both directions before any provider call. + +### Unsupported capability combinations + +No provider in this router serves search and vision together. A visual task that supplies images *and* sets `requiresSearch: true` is refused with `UnsupportedCapabilityCombinationError` (code `UNSUPPORTED_CAPABILITY_COMBINATION`) before any budget reservation, circuit permit, or provider dispatch — rather than silently dropping the images or silently skipping the search. Split such work into a vision task and a search task. + ## Vision execution Images supplied through execution options are merged into the validated task before routing. This ensures model selection and budget estimation use the same image count that reaches the provider. @@ -110,6 +157,8 @@ Only HTTPS public URLs and bounded `data:image/*;base64` payloads are accepted. For eligible high-complexity Perplexity tasks, `{ consensus: true }` executes the configured variations in parallel. The returned content is selected from the successful responses, while token and cost fields represent the aggregate successful consensus execution so budget reconciliation does not undercount spend. +Consensus is an execution modifier, not search-policy authority. It applies only to a route that already resolved to the search plane; on a general or vision route it is inert and never pulls the task onto a search provider. + ## Budget semantics The built-in tracker is process-local. diff --git a/package-lock.json b/package-lock.json index f6c34f1..53f43b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quantum-l9/llm-router", - "version": "1.1.3", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@quantum-l9/llm-router", - "version": "1.1.3", + "version": "1.2.0", "license": "PROPRIETARY", "dependencies": { "@quantum-l9/graphiti-memory-client": "^2.0.0", diff --git a/package.json b/package.json index 8ee331f..1df5f86 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@quantum-l9/llm-router", - "version": "1.1.3", + "version": "1.2.0", "type": "module", "description": "Reusable multi-provider LLM routing module with governed l9-graphiti-memory hydration, task-to-model routing, budgets, search, vision, and provider resilience.", "main": "dist/index.js", diff --git a/scripts/fixtures/declaration-consumer/consumer.ts b/scripts/fixtures/declaration-consumer/consumer.ts index a813405..67d4da6 100644 --- a/scripts/fixtures/declaration-consumer/consumer.ts +++ b/scripts/fixtures/declaration-consumer/consumer.ts @@ -1,12 +1,32 @@ -import { L9LLMRouter, TaskComplexity, TaskType, type TaskDescriptor } from '../../../dist/index.js'; +import { + L9LLMRouter, + SearchPolicySource, + TaskComplexity, + TaskType, + UnsupportedCapabilityCombinationError, + resolveSearchPolicy, + type RoutingDecision, + type SearchPolicyResolution, + type TaskDescriptor, +} from '../../../dist/index.js'; import { OpenRouterClient } from '../../../dist/providers/openrouter.js'; import { PerplexityClient } from '../../../dist/providers/perplexity.js'; import { VIEWPORTS } from '../../../dist/vision/index.js'; const task: TaskDescriptor = { type: TaskType.CLASSIFICATION, complexity: TaskComplexity.LOW, clientId: 'fixture' }; +// Capability declaration is the application's; provider and model are not. +const strategicWithoutSearch: TaskDescriptor = { type: TaskType.STRATEGIC_REASONING, complexity: TaskComplexity.HIGH, requiresSearch: false, clientId: 'fixture' }; +const freshWebWithSearch: TaskDescriptor = { type: TaskType.STRATEGIC_REASONING, complexity: TaskComplexity.HIGH, requiresSearch: true, clientId: 'fixture' }; +const policy: SearchPolicyResolution = resolveSearchPolicy(strategicWithoutSearch); +const policySource: SearchPolicySource = policy.source; +const decision: RoutingDecision | undefined = undefined; +const conflict: UnsupportedCapabilityCombinationError | undefined = undefined; const router: L9LLMRouter | undefined = undefined; const openrouter: OpenRouterClient | undefined = undefined; const perplexity: PerplexityClient | undefined = undefined; // Reference every imported symbol so the declaration build proves each public // type and value is consumable from the packaged `dist/` entry points. -export const declarationConsumerProbe = [task, router, openrouter, perplexity, VIEWPORTS] as const; +export const declarationConsumerProbe = [ + task, strategicWithoutSearch, freshWebWithSearch, policy, policySource, decision, conflict, + router, openrouter, perplexity, VIEWPORTS, +] as const; diff --git a/src/index.ts b/src/index.ts index a7ba1f2..0e860b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { import { CircuitBreaker, CircuitOpenError, type CircuitPermit } from './circuit-breaker.js'; import { resolveGeneralConfig, getFallbackChain } from './matrices/general-matrix.js'; import { resolvePerplexityConfig } from './matrices/perplexity-matrix.js'; -import { requiresSearchProvider } from './matrices/search-policy.js'; +import { resolveSearchPolicy, UnsupportedCapabilityCombinationError } from './matrices/search-policy.js'; import { classifyProviderError, isCircuitFailure } from './provider-errors.js'; import { OpenRouterClient, validateImageUrl, type OpenRouterClientLike } from './providers/openrouter.js'; import { PerplexityClient, type PerplexityClientLike } from './providers/perplexity.js'; @@ -41,16 +41,30 @@ export interface RouterDependencies { } export function resolveRoute(task: TaskDescriptor): RoutingResolution { - if (requiresSearchProvider(task)) { + const policy = resolveSearchPolicy(task); + const audit = { taskType: task.type, complexity: task.complexity, searchRequired: policy.required, searchPolicySource: policy.source }; + const imageCount = task.images?.length ?? 0; + + // Fail closed before either capability can be silently discarded. The search + // plane has no multimodal transport, so a visual task carrying images cannot + // also be answered by web search. + if (policy.required && VISION_TASKS.has(task.type) && imageCount > 0) { + throw new UnsupportedCapabilityCombinationError( + `Task[${task.type}] supplied ${imageCount} image(s) and requires search, but no provider in this router serves search and vision together. Split the work into a vision task and a search task.`, + { taskType: task.type, searchRequired: true, imageCount }, + ); + } + + if (policy.required) { const config = resolvePerplexityConfig(task); - return { taskType: task.type, complexity: task.complexity, provider: Provider.PERPLEXITY, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; + return { ...audit, provider: Provider.PERPLEXITY, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; } if (VISION_TASKS.has(task.type)) { const config = resolveVisionConfig(task.type as TaskType.VISUAL_QA | TaskType.SCREENSHOT_ANALYSIS | TaskType.LAYOUT_VALIDATION, task.complexity, task.images?.length ?? 1); - return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; + return { ...audit, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; } const config = resolveGeneralConfig(task); - return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; + return { ...audit, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; } export function getDowngradedModel( @@ -159,9 +173,18 @@ export class L9LLMRouter { images: string[] | undefined, options: { images?: string[]; assistantContext?: string; consensus?: boolean; signal?: AbortSignal } | undefined, ): Promise { + // The audited decision and the plane about to be dispatched must agree in + // both directions: a search decision may not execute on the general plane, + // and a non-search decision may not execute web search. Perplexity is the + // router's only search-capable provider. + if (decision.searchRequired !== (decision.provider === Provider.PERPLEXITY)) { + throw new Error(`Routing decision searchRequired=${decision.searchRequired} disagrees with provider ${decision.provider}`); + } if (decision.provider === Provider.PERPLEXITY) { const config = resolvePerplexityConfig(task); if (!Object.values(SonarModel).includes(decision.model as SonarModel)) throw new Error('Perplexity route resolved a non-Sonar model'); + // A search route may never dispatch a config that turns search off. + if (config.disableSearch) throw new Error('Search route resolved a Perplexity config with search disabled'); config.model = decision.model as SonarModel; if (options?.consensus && config.variations > 1) { return this.perplexity.completeWithConsensus(config, effectiveSystemPrompt, userPrompt, options.assistantContext, options.signal).then(consensus => ({ @@ -247,7 +270,12 @@ export { ProviderRequestError } from './provider-errors.js'; export { TaskValidationError, RouterConfigValidationError } from './schemas.js'; export { UnsafeImageUrlError, InvalidBaseUrlError, DEFAULT_OPENROUTER_BASE_URL, resolveOpenRouterBaseUrl } from './providers/openrouter.js'; export { VIEWPORTS } from './vision/index.js'; -export { isSearchTask, requiresSearchProvider } from './matrices/search-policy.js'; +export { + isSearchTask, + requiresSearchProvider, + resolveSearchPolicy, + UnsupportedCapabilityCombinationError, +} from './matrices/search-policy.js'; export { hydrateRouterPrompt } from './memory.js'; export type { RouterMemoryConfig } from './memory.js'; diff --git a/src/matrices/perplexity-matrix.ts b/src/matrices/perplexity-matrix.ts index c8750cf..fc6280f 100644 --- a/src/matrices/perplexity-matrix.ts +++ b/src/matrices/perplexity-matrix.ts @@ -10,11 +10,11 @@ import { type PerplexityConfig, type TaskDescriptor, } from '../types.js'; -import { isSearchTask } from './search-policy.js'; - // Re-exported for backward compatibility: `isSearchTask` historically lived in -// this module. Its canonical home is now ./search-policy.ts. -export { isSearchTask }; +// this module. Its canonical home is now ./search-policy.ts, and nothing here +// calls it any more — the search decision is made before a Perplexity config +// is ever resolved. +export { isSearchTask } from './search-policy.js'; function selectSonarModel(complexity: TaskComplexity, rank: number): SonarModel { if (complexity === TaskComplexity.CRITICAL) return SonarModel.SONAR_DEEP_RESEARCH; @@ -57,7 +57,12 @@ export function resolvePerplexityConfig(task: TaskDescriptor): PerplexityConfig domainFilter: task.domainFilter ?? [], variations, reasoningEffort: selectReasoningEffort(model, task.complexity), - disableSearch: task.requiresSearch === false && !isSearchTask(task.type), + // A Perplexity config is only ever produced for a route that resolved to + // the search plane, so search is always on. The previous predicate + // (`requiresSearch === false && !isSearchTask(type)`) was unreachable on + // that route and, off-route, produced a search-provider config with search + // disabled — a config that contradicted the decision it belonged to. + disableSearch: false, estimatedCostPerCall, resolutionReason: `Task[${task.type}] complexity[${task.complexity}] uses ${model}`, }; diff --git a/src/matrices/search-policy.ts b/src/matrices/search-policy.ts index 182b613..f5cc68f 100644 --- a/src/matrices/search-policy.ts +++ b/src/matrices/search-policy.ts @@ -1,4 +1,4 @@ -import { TaskType, type TaskDescriptor } from '../types.js'; +import { SearchPolicySource, TaskType, type SearchPolicyResolution, type TaskDescriptor } from '../types.js'; /** * Task types whose *default* capability implies a search-backed provider. @@ -20,14 +20,15 @@ const DEFAULT_SEARCH_TASKS = new Set([ * * Preserved verbatim so existing consumers that reason purely about a * `TaskType` keep the same answer. New routing decisions should prefer - * {@link requiresSearchProvider}, which honours an explicit capability flag. + * {@link resolveSearchPolicy}, which honours an explicit capability flag and + * reports *why* the answer was reached. */ export function isSearchTask(type: TaskType): boolean { return DEFAULT_SEARCH_TASKS.has(type); } /** - * Explicit capability declaration wins. + * Canonical search-policy resolver — the single implementation of the rule. * * Applications declare *whether the task needs a search provider* via * `TaskDescriptor.requiresSearch`. When present, that declaration is @@ -38,13 +39,51 @@ export function isSearchTask(type: TaskType): boolean { * caller already had normalized evidence and explicitly did not require search. * * Semantics: - * requiresSearch === true -> search provider - * requiresSearch === false -> general reasoning provider - * requiresSearch === undefined -> legacy TaskType default (isSearchTask) + * requiresSearch === true -> search provider (source EXPLICIT) + * requiresSearch === false -> general provider (source EXPLICIT) + * requiresSearch === undefined -> isSearchTask(type) (source TASK_DEFAULT) + * + * The returned `source` is what makes an audited routing decision provable: + * it distinguishes "the caller asked for this" from "the task type implied it". */ -export function requiresSearchProvider(task: TaskDescriptor): boolean { +export function resolveSearchPolicy(task: TaskDescriptor): SearchPolicyResolution { if (typeof task.requiresSearch === 'boolean') { - return task.requiresSearch; + return { required: task.requiresSearch, source: SearchPolicySource.EXPLICIT }; + } + return { required: isSearchTask(task.type), source: SearchPolicySource.TASK_DEFAULT }; +} + +/** + * Boolean view of {@link resolveSearchPolicy}. Retained as the 1.x public + * predicate; it delegates so there is exactly one implementation of the rule. + */ +export function requiresSearchProvider(task: TaskDescriptor): boolean { + return resolveSearchPolicy(task).required; +} + +/** + * Fail-closed error for a task that asks for two capabilities the router has no + * provider contract able to satisfy together. + * + * Raised today for `vision task type + images + requiresSearch === true`: the + * search plane (Perplexity Sonar) has no multimodal transport in this router, + * so honouring one capability necessarily discards the other. Dropping either + * silently would make the routing audit a lie, so the request is rejected + * before any reservation, circuit permit, or provider dispatch. + * + * This is a caller-side contract error, not a provider failure: it must never + * count against provider circuit health. + */ +export class UnsupportedCapabilityCombinationError extends Error { + public readonly code = 'UNSUPPORTED_CAPABILITY_COMBINATION'; + constructor( + message: string, + public readonly requested: Readonly<{ taskType: TaskType; searchRequired: boolean; imageCount: number }>, + ) { + super(message); + this.name = 'UnsupportedCapabilityCombinationError'; + } + toJSON(): Record { + return { name: this.name, code: this.code, message: this.message, requested: this.requested }; } - return isSearchTask(task.type); } diff --git a/src/types.ts b/src/types.ts index ddbd12e..b5154d7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -33,6 +33,19 @@ export enum GeneralModel { GEMINI_FLASH_VISION = 'google/gemini-2.5-flash', } +/** + * Why a routing decision required (or did not require) a search provider. + * + * EXPLICIT — the caller set `TaskDescriptor.requiresSearch` to a boolean. + * TASK_DEFAULT — the caller left it undefined, so the `TaskType` default applied. + */ +export enum SearchPolicySource { EXPLICIT = 'explicit', TASK_DEFAULT = 'task_default' } + +export interface SearchPolicyResolution { + required: boolean; + source: SearchPolicySource; +} + export enum SearchContextSize { LOW = 'low', MEDIUM = 'medium', HIGH = 'high' } export enum SearchMode { WEB = 'web', ACADEMIC = 'academic', SEC = 'sec' } export enum RecencyFilter { HOUR = 'hour', DAY = 'day', WEEK = 'week', MONTH = 'month', YEAR = 'year', NONE = 'none' } @@ -190,6 +203,10 @@ export interface RoutingResolution { model: GeneralModel | SonarModel; estimatedCost: number; reason: string; + /** Whether this decision resolved to a search-capable provider plane. */ + searchRequired: boolean; + /** Whether `searchRequired` came from the caller or from the TaskType default. */ + searchPolicySource: SearchPolicySource; } export interface RoutingDecision extends RoutingResolution { diff --git a/tests/routing-matrix.test.ts b/tests/routing-matrix.test.ts new file mode 100644 index 0000000..36faa36 --- /dev/null +++ b/tests/routing-matrix.test.ts @@ -0,0 +1,142 @@ +import { describe, expect, it } from 'vitest'; +import { + Provider, + SearchPolicySource, + SonarModel, + TaskComplexity, + TaskType, + type GeneralModel, + type TaskDescriptor, +} from '../src/types.js'; +import { L9LLMRouter, resolveRoute, UnsupportedCapabilityCombinationError } from '../src/index.js'; +import { resolveGeneralConfig } from '../src/matrices/general-matrix.js'; +import { resolveVisionConfig } from '../src/vision/index.js'; + +/** + * Route planes are distinguished by provider *and* by which resolver owns the + * model, because the vision plane and the general plane share Provider.OPENROUTER. + */ +type Plane = 'SEARCH' | 'NON_SEARCH' | 'VISION' | 'FAIL_CLOSED'; + +const IMAGES = ['https://cdn.example.com/shot.png']; + +interface MatrixCase { + id: string; + task: TaskDescriptor; + expected: Plane; + expectedSource: SearchPolicySource; +} + +const task = (over: Partial & Pick): TaskDescriptor => ({ + complexity: TaskComplexity.MEDIUM, + clientId: 'matrix-client', + ...over, +}); + +const MATRIX: MatrixCase[] = [ + { id: 'A STRATEGIC_REASONING requiresSearch=false', task: task({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), expected: 'NON_SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'B STRATEGIC_REASONING requiresSearch=true', task: task({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true }), expected: 'SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'C STRATEGIC_REASONING requiresSearch=undefined', task: task({ type: TaskType.STRATEGIC_REASONING }), expected: 'NON_SEARCH', expectedSource: SearchPolicySource.TASK_DEFAULT }, + { id: 'D COMPETITOR_RESEARCH requiresSearch=true', task: task({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: true }), expected: 'SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'E COMPETITOR_RESEARCH requiresSearch=false', task: task({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false }), expected: 'NON_SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'F COMPETITOR_RESEARCH requiresSearch=undefined', task: task({ type: TaskType.COMPETITOR_RESEARCH }), expected: 'SEARCH', expectedSource: SearchPolicySource.TASK_DEFAULT }, + { id: 'G FACT_VERIFICATION requiresSearch=false', task: task({ type: TaskType.FACT_VERIFICATION, requiresSearch: false }), expected: 'NON_SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'H FACT_VERIFICATION requiresSearch=undefined', task: task({ type: TaskType.FACT_VERIFICATION }), expected: 'SEARCH', expectedSource: SearchPolicySource.TASK_DEFAULT }, + { id: 'I CONTENT_GENERATION requiresSearch=false', task: task({ type: TaskType.CONTENT_GENERATION, requiresSearch: false }), expected: 'NON_SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'J CONTENT_GENERATION requiresSearch=true', task: task({ type: TaskType.CONTENT_GENERATION, requiresSearch: true }), expected: 'SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'K SCREENSHOT_ANALYSIS+imgs requiresSearch=false', task: task({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: false }), expected: 'VISION', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'L SCREENSHOT_ANALYSIS+imgs requiresSearch=undefined', task: task({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES }), expected: 'VISION', expectedSource: SearchPolicySource.TASK_DEFAULT }, + { id: 'M SCREENSHOT_ANALYSIS+imgs requiresSearch=true', task: task({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: true }), expected: 'FAIL_CLOSED', expectedSource: SearchPolicySource.EXPLICIT }, + // Extra coverage required by §5: explicit true lifts otherwise-general task types. + { id: 'N EXTRACTION requiresSearch=true', task: task({ type: TaskType.EXTRACTION, requiresSearch: true }), expected: 'SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'O CLASSIFICATION requiresSearch=true', task: task({ type: TaskType.CLASSIFICATION, requiresSearch: true }), expected: 'SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'P MARKET_RESEARCH requiresSearch=false', task: task({ type: TaskType.MARKET_RESEARCH, requiresSearch: false }), expected: 'NON_SEARCH', expectedSource: SearchPolicySource.EXPLICIT }, + { id: 'Q MARKET_RESEARCH requiresSearch=undefined', task: task({ type: TaskType.MARKET_RESEARCH }), expected: 'SEARCH', expectedSource: SearchPolicySource.TASK_DEFAULT }, +]; + +function planeOf(descriptor: TaskDescriptor): Plane { + const decision = resolveRoute(descriptor); + if (decision.provider === Provider.PERPLEXITY) { + expect(Object.values(SonarModel)).toContain(decision.model); + return 'SEARCH'; + } + const vision = VISION_TYPES.has(descriptor.type) + ? resolveVisionConfig( + descriptor.type as TaskType.VISUAL_QA | TaskType.SCREENSHOT_ANALYSIS | TaskType.LAYOUT_VALIDATION, + descriptor.complexity, + descriptor.images?.length ?? 1, + ) + : undefined; + if (vision && decision.model === vision.model && decision.estimatedCost === vision.estimatedCostPerCall) return 'VISION'; + const general = resolveGeneralConfig(descriptor); + expect(decision.model).toBe(general.model); + return 'NON_SEARCH'; +} + +const VISION_TYPES = new Set([TaskType.VISUAL_QA, TaskType.SCREENSHOT_ANALYSIS, TaskType.LAYOUT_VALIDATION]); + +describe('§16 routing matrix — explicit requiresSearch is authoritative', () => { + it.each(MATRIX)('$id -> $expected', ({ task: descriptor, expected }) => { + if (expected === 'FAIL_CLOSED') { + expect(() => resolveRoute(descriptor)).toThrow(UnsupportedCapabilityCombinationError); + return; + } + expect(planeOf(descriptor)).toBe(expected); + }); + + it('resolves every matrix case identically through the public router', () => { + const router = new L9LLMRouter({ perplexityApiKey: 'p', openrouterApiKey: 'o' }, { idFactory: () => 'id', clock: () => new Date('2026-01-01T00:00:00Z') }); + for (const entry of MATRIX) { + if (entry.expected === 'FAIL_CLOSED') { + expect(() => router.route(entry.task), entry.id).toThrow(UnsupportedCapabilityCombinationError); + continue; + } + const viaRouter = router.route(entry.task); + const direct = resolveRoute(entry.task); + expect({ provider: viaRouter.provider, model: viaRouter.model }, entry.id).toEqual({ provider: direct.provider, model: direct.model }); + } + }); +}); + +describe('§17 routing audit — searchRequired and searchPolicySource', () => { + it.each(MATRIX.filter(entry => entry.expected !== 'FAIL_CLOSED'))( + '$id exposes provable search-policy evidence', + ({ task: descriptor, expected, expectedSource }) => { + const decision = resolveRoute(descriptor); + expect(decision.searchPolicySource).toBe(expectedSource); + expect(decision.searchRequired).toBe(expected === 'SEARCH'); + // The audited flag must never disagree with the plane actually selected. + expect(decision.searchRequired).toBe(decision.provider === Provider.PERPLEXITY); + }, + ); + + it('carries the audit through the full RoutingDecision surface', () => { + const router = new L9LLMRouter({ perplexityApiKey: 'p', openrouterApiKey: 'o' }, { idFactory: () => 'task-1', clock: () => new Date('2026-01-01T00:00:00Z') }); + const explicitFalse = router.route(task({ type: TaskType.MARKET_RESEARCH, requiresSearch: false })); + expect(explicitFalse).toMatchObject({ + taskId: 'task-1', + clientId: 'matrix-client', + timestamp: '2026-01-01T00:00:00.000Z', + taskType: TaskType.MARKET_RESEARCH, + complexity: TaskComplexity.MEDIUM, + provider: Provider.OPENROUTER, + searchRequired: false, + searchPolicySource: SearchPolicySource.EXPLICIT, + }); + expect(typeof explicitFalse.reason).toBe('string'); + expect(explicitFalse.estimatedCost).toBeGreaterThan(0); + + const explicitTrue = router.route(task({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true })); + expect(explicitTrue).toMatchObject({ provider: Provider.PERPLEXITY, searchRequired: true, searchPolicySource: SearchPolicySource.EXPLICIT }); + + const omitted = router.route(task({ type: TaskType.COMPETITOR_RESEARCH })); + expect(omitted).toMatchObject({ provider: Provider.PERPLEXITY, searchRequired: true, searchPolicySource: SearchPolicySource.TASK_DEFAULT }); + }); + + it('never leaks credentials or prompts into the routing audit', () => { + const router = new L9LLMRouter({ perplexityApiKey: 'pplx-secret', openrouterApiKey: 'or-secret' }, { idFactory: () => 'id' }); + const serialized = JSON.stringify(router.route(task({ type: TaskType.FACT_VERIFICATION }))); + expect(serialized).not.toContain('pplx-secret'); + expect(serialized).not.toContain('or-secret'); + }); +}); diff --git a/tests/search-policy-dispatch.test.ts b/tests/search-policy-dispatch.test.ts new file mode 100644 index 0000000..644f8b6 --- /dev/null +++ b/tests/search-policy-dispatch.test.ts @@ -0,0 +1,305 @@ +import { describe, expect, it } from 'vitest'; +import { + GeneralModel, + Provider, + SearchPolicySource, + SonarModel, + TaskComplexity, + TaskType, + type GeneralModelConfig, + type LLMResponse, + type PerplexityConfig, + type VisionConfig, +} from '../src/types.js'; +import { L9LLMRouter, UnsupportedCapabilityCombinationError } from '../src/index.js'; +import { ProviderRequestError } from '../src/provider-errors.js'; +import { resolvePerplexityConfig } from '../src/matrices/perplexity-matrix.js'; +import { buildRequestBody } from '../src/providers/perplexity.js'; +import { resolveVisionConfig } from '../src/vision/index.js'; + +const response: LLMResponse = { + content: 'ok', model: GeneralModel.GPT4O_MINI, provider: Provider.OPENROUTER, + inputTokens: 1, outputTokens: 1, totalTokens: 2, cost: 0.1, latencyMs: 5, cached: false, +}; + +interface Calls { + general: GeneralModelConfig[]; + vision: VisionConfig[]; + search: PerplexityConfig[]; + consensus: PerplexityConfig[]; +} + +function harness() { + const calls: Calls = { general: [], vision: [], search: [], consensus: [] }; + const openrouterClient = { + complete: async (config: GeneralModelConfig) => { calls.general.push(config); return response; }, + completeWithFallback: async (config: GeneralModelConfig) => { calls.general.push(config); return { ...response, model: config.model }; }, + completeWithVision: async (config: VisionConfig) => { calls.vision.push(config); return { ...response, model: config.model }; }, + }; + const perplexityClient = { + complete: async (config: PerplexityConfig) => { calls.search.push(config); return { ...response, model: config.model, provider: Provider.PERPLEXITY }; }, + completeWithConsensus: async (config: PerplexityConfig) => { + calls.consensus.push(config); + return { + best: { ...response, model: config.model, provider: Provider.PERPLEXITY }, + all: [], consensusScore: 1, + aggregate: { inputTokens: 1, outputTokens: 1, totalTokens: 2, cost: 0.1, latencyMs: 5, citations: [] }, + }; + }, + }; + const router = new L9LLMRouter( + { perplexityApiKey: 'p', openrouterApiKey: 'o' }, + { openrouterClient, perplexityClient, idFactory: () => 'task-1', clock: () => new Date('2026-01-01T00:00:00Z') }, + ); + router.initClient('c'); + return { router, calls }; +} + +describe('§18 provider dispatch follows the resolved search policy', () => { + it('requiresSearch=true on a general TaskType dispatches the search client only', async () => { + const { router, calls } = harness(); + const result = await router.execute( + { clientId: 'c', type: TaskType.STRATEGIC_REASONING, complexity: TaskComplexity.MEDIUM, requiresSearch: true }, + 's', 'u', + ); + expect(calls.search).toHaveLength(1); + expect(calls.general).toHaveLength(0); + expect(calls.vision).toHaveLength(0); + expect(result.provider).toBe(Provider.PERPLEXITY); + expect(router.getCallLog()[0]).toMatchObject({ + provider: Provider.PERPLEXITY, searchRequired: true, searchPolicySource: SearchPolicySource.EXPLICIT, + }); + }); + + it('requiresSearch=false on a default-search TaskType dispatches the general client only', async () => { + const { router, calls } = harness(); + const result = await router.execute( + { clientId: 'c', type: TaskType.COMPETITOR_RESEARCH, complexity: TaskComplexity.HIGH, requiresSearch: false }, + 's', 'u', + ); + expect(calls.general).toHaveLength(1); + expect(calls.search).toHaveLength(0); + expect(calls.consensus).toHaveLength(0); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(router.getCallLog()[0]).toMatchObject({ + provider: Provider.OPENROUTER, searchRequired: false, searchPolicySource: SearchPolicySource.EXPLICIT, + }); + }); + + it('omitted requiresSearch keeps the TaskType default plane and records TASK_DEFAULT', async () => { + const { router, calls } = harness(); + await router.execute({ clientId: 'c', type: TaskType.FACT_VERIFICATION, complexity: TaskComplexity.MEDIUM }, 's', 'u'); + expect(calls.search).toHaveLength(1); + expect(router.getCallLog()[0]).toMatchObject({ searchRequired: true, searchPolicySource: SearchPolicySource.TASK_DEFAULT }); + }); + + it('a visual task dispatches the vision path with its images intact', async () => { + const { router, calls } = harness(); + await router.execute( + { clientId: 'c', type: TaskType.SCREENSHOT_ANALYSIS, complexity: TaskComplexity.MEDIUM, requiresSearch: false }, + 's', 'u', { images: ['https://cdn.example.com/a.png'] }, + ); + expect(calls.vision).toHaveLength(1); + expect(calls.search).toHaveLength(0); + expect(router.getCallLog()[0]).toMatchObject({ searchRequired: false, searchPolicySource: SearchPolicySource.EXPLICIT }); + }); + + it('the audited model matches the model actually dispatched', async () => { + const { router, calls } = harness(); + await router.execute({ clientId: 'c', type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.HIGH }, 's', 'u'); + expect(calls.search[0].model).toBe(router.getCallLog()[0].model); + }); +}); + +describe('§6 search + vision fails closed instead of losing a capability', () => { + it('rejects a visual task carrying images that also requires search', async () => { + const { router, calls } = harness(); + await expect(router.execute( + { clientId: 'c', type: TaskType.SCREENSHOT_ANALYSIS, complexity: TaskComplexity.MEDIUM, requiresSearch: true }, + 's', 'u', { images: ['https://cdn.example.com/a.png'] }, + )).rejects.toBeInstanceOf(UnsupportedCapabilityCombinationError); + // No capability was silently chosen, and no provider was touched. + expect(calls.search).toHaveLength(0); + expect(calls.vision).toHaveLength(0); + expect(calls.general).toHaveLength(0); + }); + + it('carries a stable machine-readable code and the requested capabilities', async () => { + const { router } = harness(); + const error = await router.execute( + { clientId: 'c', type: TaskType.VISUAL_QA, complexity: TaskComplexity.HIGH, requiresSearch: true, images: ['https://cdn.example.com/a.png', 'https://cdn.example.com/b.png'] }, + 's', 'u', + ).catch((caught: unknown) => caught as UnsupportedCapabilityCombinationError); + expect(error).toBeInstanceOf(UnsupportedCapabilityCombinationError); + expect(error.code).toBe('UNSUPPORTED_CAPABILITY_COMBINATION'); + expect(error.name).toBe('UnsupportedCapabilityCombinationError'); + expect(error.requested).toEqual({ taskType: TaskType.VISUAL_QA, searchRequired: true, imageCount: 2 }); + }); + + it('reserves no budget and opens no circuit when the combination is refused', async () => { + const { router } = harness(); + await expect(router.execute( + { clientId: 'c', type: TaskType.LAYOUT_VALIDATION, complexity: TaskComplexity.MEDIUM, requiresSearch: true, images: ['https://cdn.example.com/a.png'] }, + 's', 'u', + )).rejects.toThrow(UnsupportedCapabilityCombinationError); + expect(router.getClientBudgetReport('c')).toMatchObject({ monthSpend: 0, reservedSpend: 0, activeReservations: 0 }); + expect(router.getCircuitState(Provider.PERPLEXITY).failureCount).toBe(0); + expect(router.getCircuitState(Provider.OPENROUTER).failureCount).toBe(0); + expect(router.getCallLog()).toHaveLength(0); + }); + + it('leaves vision model selection for a given image count exactly as it was', () => { + const { router } = harness(); + // Regression guard: the conflict check must not perturb the image count the + // vision matrix sees, including the empty-array edge case. + for (const images of [undefined, [], ['https://cdn.example.com/a.png'], ['https://cdn.example.com/a.png', 'https://cdn.example.com/b.png']]) { + for (const complexity of Object.values(TaskComplexity)) { + const decision = router.route({ clientId: 'c', type: TaskType.SCREENSHOT_ANALYSIS, complexity, images }); + const expected = resolveVisionConfig(TaskType.SCREENSHOT_ANALYSIS, complexity, images?.length ?? 1); + expect({ model: decision.model, cost: decision.estimatedCost }).toEqual({ model: expected.model, cost: expected.estimatedCostPerCall }); + } + } + }); + + it('a visual TaskType with no images and explicit search is a plain search request', async () => { + const { router, calls } = harness(); + await router.execute( + { clientId: 'c', type: TaskType.SCREENSHOT_ANALYSIS, complexity: TaskComplexity.MEDIUM, requiresSearch: true }, + 's', 'u', + ); + // Nothing visual was supplied, so nothing visual is discarded. + expect(calls.search).toHaveLength(1); + expect(calls.vision).toHaveLength(0); + }); +}); + +describe('§7 Perplexity config agrees with the routing decision', () => { + it('never resolves a search config with search disabled', () => { + for (const type of Object.values(TaskType)) { + for (const complexity of Object.values(TaskComplexity)) { + for (const requiresSearch of [true, false, undefined]) { + expect(resolvePerplexityConfig({ type, complexity, requiresSearch, clientId: 'c' }).disableSearch).toBe(false); + } + } + } + }); + + it('emits web_search_options for every resolved search route', () => { + const config = resolvePerplexityConfig({ type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.HIGH, clientId: 'c' }); + expect(buildRequestBody(config, []).web_search_options).toBeDefined(); + }); + + it('rejects dispatch if a decision and its plane ever disagree', () => { + const { router } = harness(); + // Force the contradiction the invariant exists to catch. + const contradictory = { ...router.route({ clientId: 'c', type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.HIGH }), searchRequired: false }; + expect(contradictory.provider).toBe(Provider.PERPLEXITY); + const dispatch = Reflect.get(router, 'dispatchProvider') as (...args: unknown[]) => Promise; + expect(() => dispatch.call( + router, + { clientId: 'c', type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.HIGH }, + contradictory, 's', 'u', undefined, undefined, + )).toThrow(/disagrees with provider/); + }); +}); + +describe('§14 consensus is an execution modifier, not search-policy authority', () => { + it('does not let consensus=true pull a non-search task onto the search plane', async () => { + const { router, calls } = harness(); + const result = await router.execute( + { clientId: 'c', type: TaskType.COMPETITOR_RESEARCH, complexity: TaskComplexity.HIGH, requiresSearch: false }, + 's', 'u', { consensus: true }, + ); + expect(calls.consensus).toHaveLength(0); + expect(calls.search).toHaveLength(0); + expect(calls.general).toHaveLength(1); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(router.getCallLog()[0]).toMatchObject({ searchRequired: false, searchPolicySource: SearchPolicySource.EXPLICIT }); + }); + + it('still applies consensus on an actually-selected search route', async () => { + const { router, calls } = harness(); + await router.execute( + { clientId: 'c', type: TaskType.STRATEGIC_REASONING, complexity: TaskComplexity.HIGH, requiresSearch: true }, + 's', 'u', { consensus: true }, + ); + expect(calls.consensus).toHaveLength(1); + expect(calls.consensus[0].variations).toBeGreaterThan(1); + }); +}); + +describe('§11/§12 resilience is not bypassed by the search-policy path', () => { + it('reserves budget before dispatching either plane', async () => { + for (const requiresSearch of [true, false]) { + const { router } = harness(); + const reserved: number[] = []; + const store = Reflect.get(router, 'budgetStore') as { reserveTask: (...args: never[]) => Promise }; + const original = store.reserveTask.bind(store); + store.reserveTask = async (...args: never[]) => { reserved.push(1); return original(...args); }; + await router.execute({ clientId: 'c', type: TaskType.STRATEGIC_REASONING, complexity: TaskComplexity.MEDIUM, requiresSearch }, 's', 'u'); + expect(reserved).toHaveLength(1); + expect(router.getClientBudgetReport('c')).toMatchObject({ monthSpend: 0.1, reservedSpend: 0, activeReservations: 0 }); + } + }); + + it('charges failures to the circuit of the provider the policy selected', async () => { + const down = (provider: Provider) => new ProviderRequestError(`${provider} down`, { provider, kind: 'server', retryable: true, status: 503 }); + const failing = new L9LLMRouter( + { perplexityApiKey: 'p', openrouterApiKey: 'o', circuitBreaker: { failureThreshold: 1 } }, + { + openrouterClient: { + complete: async () => response, + completeWithVision: async () => response, + completeWithFallback: async () => { throw down(Provider.OPENROUTER); }, + }, + perplexityClient: { + complete: async () => { throw down(Provider.PERPLEXITY); }, + completeWithConsensus: async () => { throw down(Provider.PERPLEXITY); }, + }, + }, + ); + failing.initClient('c'); + + await expect(failing.execute({ clientId: 'c', type: TaskType.COMPETITOR_RESEARCH, complexity: TaskComplexity.LOW, requiresSearch: true }, 's', 'u')).rejects.toThrow(); + expect(failing.getCircuitState(Provider.PERPLEXITY).state).toBe('open'); + expect(failing.getCircuitState(Provider.OPENROUTER).state).toBe('closed'); + + await expect(failing.execute({ clientId: 'c', type: TaskType.COMPETITOR_RESEARCH, complexity: TaskComplexity.LOW, requiresSearch: false }, 's', 'u')).rejects.toThrow(); + expect(failing.getCircuitState(Provider.OPENROUTER).state).toBe('open'); + }); + + it('keeps image safety validation ahead of every routing outcome', async () => { + const { router, calls } = harness(); + await expect(router.execute( + { clientId: 'c', type: TaskType.SCREENSHOT_ANALYSIS, complexity: TaskComplexity.MEDIUM, requiresSearch: false }, + 's', 'u', { images: ['https://127.0.0.1/a.png'] }, + )).rejects.toThrow(/private/); + expect(calls.vision).toHaveLength(0); + expect(router.getCircuitState(Provider.OPENROUTER).failureCount).toBe(0); + }); + + it('a downgraded search route stays on the search plane', async () => { + const { router, calls } = harness(); + await router.execute({ clientId: 'c', type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.CRITICAL }, 's', 'u'); + const logged = router.getCallLog()[0]; + expect(Object.values(SonarModel)).toContain(logged.model); + expect(logged.searchRequired).toBe(true); + expect(calls.general).toHaveLength(0); + }); +}); + +describe('§23 invalid search policy is a validation error, not a warning', () => { + it('rejects a non-boolean requiresSearch before routing', () => { + const { router } = harness(); + expect(() => router.route({ type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.LOW, requiresSearch: 'yes' } as never)) + .toThrow(/Invalid TaskDescriptor/); + }); + + it('still refuses application-selected provider and model fields', () => { + const { router } = harness(); + const decision = router.route({ type: TaskType.MARKET_RESEARCH, complexity: TaskComplexity.LOW, provider: Provider.OPENROUTER, model: GeneralModel.GPT4O } as never); + // Unknown keys are stripped by the schema; the router keeps provider authority. + expect(decision.provider).toBe(Provider.PERPLEXITY); + expect(Object.values(SonarModel)).toContain(decision.model); + }); +}); diff --git a/tests/search-policy.test.ts b/tests/search-policy.test.ts index 620a139..102feb5 100644 --- a/tests/search-policy.test.ts +++ b/tests/search-policy.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { Provider, TaskComplexity, TaskType, type TaskDescriptor } from '../src/types.js'; -import { isSearchTask, requiresSearchProvider, resolveRoute } from '../src/index.js'; +import { Provider, SearchPolicySource, TaskComplexity, TaskType, type TaskDescriptor } from '../src/types.js'; +import { isSearchTask, requiresSearchProvider, resolveRoute, resolveSearchPolicy } from '../src/index.js'; const base = (over: Partial): TaskDescriptor => ({ type: TaskType.MARKET_RESEARCH, @@ -46,3 +46,31 @@ describe('search policy — explicit capability wins', () => { expect(requiresSearchProvider(base({ type: TaskType.CONTENT_GENERATION, requiresSearch: true }))).toBe(true); }); }); + +describe('search policy — canonical resolver reports its own authority', () => { + it('marks a boolean declaration EXPLICIT in both directions', () => { + expect(resolveSearchPolicy(base({ requiresSearch: true }))).toEqual({ required: true, source: SearchPolicySource.EXPLICIT }); + expect(resolveSearchPolicy(base({ requiresSearch: false }))).toEqual({ required: false, source: SearchPolicySource.EXPLICIT }); + }); + + it('marks an omitted declaration TASK_DEFAULT and defers to the TaskType', () => { + expect(resolveSearchPolicy(base({ type: TaskType.MARKET_RESEARCH }))).toEqual({ required: true, source: SearchPolicySource.TASK_DEFAULT }); + expect(resolveSearchPolicy(base({ type: TaskType.STRATEGIC_REASONING }))).toEqual({ required: false, source: SearchPolicySource.TASK_DEFAULT }); + }); + + it('keeps requiresSearchProvider as a pure view of the canonical resolver', () => { + for (const type of Object.values(TaskType)) { + for (const requiresSearch of [true, false, undefined]) { + const descriptor = base({ type, requiresSearch }); + expect(requiresSearchProvider(descriptor)).toBe(resolveSearchPolicy(descriptor).required); + } + } + }); + + it('treats a non-boolean requiresSearch as absent rather than truthy', () => { + // Defence in depth: the schema rejects these before routing, but the policy + // itself must never coerce a string into a capability grant. + expect(resolveSearchPolicy(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: 'true' as never }))) + .toEqual({ required: false, source: SearchPolicySource.TASK_DEFAULT }); + }); +});