diff --git a/src/index.ts b/src/index.ts index a7ba1f2..c73851a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -10,7 +10,7 @@ import { import { CircuitBreaker, CircuitOpenError, type CircuitPermit } from './circuit-breaker.js'; import { resolveGeneralConfig, getFallbackChain } from './matrices/general-matrix.js'; import { resolvePerplexityConfig } from './matrices/perplexity-matrix.js'; -import { requiresSearchProvider } from './matrices/search-policy.js'; +import { assertSearchVisionCompatible, requiresSearchProvider } from './matrices/search-policy.js'; import { classifyProviderError, isCircuitFailure } from './provider-errors.js'; import { OpenRouterClient, validateImageUrl, type OpenRouterClientLike } from './providers/openrouter.js'; import { PerplexityClient, type PerplexityClientLike } from './providers/perplexity.js'; @@ -25,6 +25,7 @@ import { type RouterConfig, type RoutingDecision, type RoutingResolution, + type SearchPolicySource, type TaskDescriptor, } from './types.js'; import { generateFullSiteQAPlan, resolveVisionConfig, VIEWPORTS, type FullSiteQAConfig, type VisualQATask } from './vision/index.js'; @@ -41,16 +42,19 @@ export interface RouterDependencies { } export function resolveRoute(task: TaskDescriptor): RoutingResolution { - if (requiresSearchProvider(task)) { + const searchRequired = requiresSearchProvider(task); + const searchPolicySource: SearchPolicySource = typeof task.requiresSearch === 'boolean' ? 'EXPLICIT' : 'TASK_DEFAULT'; + assertSearchVisionCompatible(task, searchRequired); + if (searchRequired) { const config = resolvePerplexityConfig(task); - return { taskType: task.type, complexity: task.complexity, provider: Provider.PERPLEXITY, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; + return { taskType: task.type, complexity: task.complexity, provider: Provider.PERPLEXITY, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason, searchRequired, searchPolicySource }; } if (VISION_TASKS.has(task.type)) { const config = resolveVisionConfig(task.type as TaskType.VISUAL_QA | TaskType.SCREENSHOT_ANALYSIS | TaskType.LAYOUT_VALIDATION, task.complexity, task.images?.length ?? 1); - return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; + return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason, searchRequired: false, searchPolicySource }; } const config = resolveGeneralConfig(task); - return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason }; + return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason, searchRequired: false, searchPolicySource }; } export function getDowngradedModel( @@ -247,7 +251,7 @@ export { ProviderRequestError } from './provider-errors.js'; export { TaskValidationError, RouterConfigValidationError } from './schemas.js'; export { UnsafeImageUrlError, InvalidBaseUrlError, DEFAULT_OPENROUTER_BASE_URL, resolveOpenRouterBaseUrl } from './providers/openrouter.js'; export { VIEWPORTS } from './vision/index.js'; -export { isSearchTask, requiresSearchProvider } from './matrices/search-policy.js'; +export { isSearchTask, requiresSearchProvider, UnsupportedCapabilityCombinationError, assertSearchVisionCompatible } from './matrices/search-policy.js'; export { hydrateRouterPrompt } from './memory.js'; export type { RouterMemoryConfig } from './memory.js'; diff --git a/src/matrices/perplexity-matrix.ts b/src/matrices/perplexity-matrix.ts index c8750cf..fa5fbb5 100644 --- a/src/matrices/perplexity-matrix.ts +++ b/src/matrices/perplexity-matrix.ts @@ -57,7 +57,11 @@ export function resolvePerplexityConfig(task: TaskDescriptor): PerplexityConfig domainFilter: task.domainFilter ?? [], variations, reasoningEffort: selectReasoningEffort(model, task.complexity), - disableSearch: task.requiresSearch === false && !isSearchTask(task.type), + // A Perplexity route is only selected when requiresSearchProvider(task) is + // true (search required by explicit flag or TaskType default), so a resolved + // router config never disables search. The field stays on the config + // contract for direct-provider callers; the router always sets false. + disableSearch: false, estimatedCostPerCall, resolutionReason: `Task[${task.type}] complexity[${task.complexity}] uses ${model}`, }; diff --git a/src/matrices/search-policy.ts b/src/matrices/search-policy.ts index 182b613..cb072ca 100644 --- a/src/matrices/search-policy.ts +++ b/src/matrices/search-policy.ts @@ -48,3 +48,33 @@ export function requiresSearchProvider(task: TaskDescriptor): boolean { } return isSearchTask(task.type); } + +/** + * Fail-closed guard for capability combinations no current provider supports. + * + * The Perplexity client is text-only, so a search-required route cannot consume + * images. Rather than silently dropping either requested capability (dropping + * the images, or routing vision while pretending search happened), resolution + * refuses the combination until a multimodal-search provider contract exists. + */ +export class UnsupportedCapabilityCombinationError extends Error { + public readonly code = 'SEARCH_VISION_COMBINATION_UNSUPPORTED' as const; + constructor(public readonly taskType: TaskType, public readonly capabilities: readonly string[]) { + super(`Task type "${taskType}" requires an unsupported capability combination: ${capabilities.join(' + ')}`); + this.name = 'UnsupportedCapabilityCombinationError'; + } + toJSON(): Record { + return { name: this.name, code: this.code, taskType: this.taskType, capabilities: this.capabilities, message: this.message }; + } +} + +/** + * Throws {@link UnsupportedCapabilityCombinationError} when a search-required + * route would also have to consume images. No silent capability loss: neither + * the images nor the search request is dropped behind the caller's back. + */ +export function assertSearchVisionCompatible(task: TaskDescriptor, searchRequired: boolean): void { + if (searchRequired && (task.images?.length ?? 0) > 0) { + throw new UnsupportedCapabilityCombinationError(task.type, ['search', 'vision']); + } +} diff --git a/src/provider-errors.ts b/src/provider-errors.ts index 22f1812..6acb644 100644 --- a/src/provider-errors.ts +++ b/src/provider-errors.ts @@ -4,6 +4,7 @@ const LOCAL_ERROR_NAMES = new Set([ 'TaskValidationError', 'RouterConfigValidationError', 'UnsafeImageUrlError', + 'UnsupportedCapabilityCombinationError', 'BudgetExhaustedError', 'CircuitOpenError', 'AbortError', diff --git a/src/types.ts b/src/types.ts index ddbd12e..044b780 100644 --- a/src/types.ts +++ b/src/types.ts @@ -183,6 +183,8 @@ export interface RouterConfig { providerMaxRetries?: 0; } +export type SearchPolicySource = 'EXPLICIT' | 'TASK_DEFAULT'; + export interface RoutingResolution { taskType: TaskType; complexity: TaskComplexity; @@ -190,6 +192,10 @@ export interface RoutingResolution { model: GeneralModel | SonarModel; estimatedCost: number; reason: string; + /** Whether the resolved route carries web-search capability (explicit flag or task-type default). */ + searchRequired: boolean; + /** Where `searchRequired` came from: an explicit `TaskDescriptor.requiresSearch` boolean, or the legacy per-TaskType default. */ + searchPolicySource: SearchPolicySource; } export interface RoutingDecision extends RoutingResolution { diff --git a/tests/routing-matrix.test.ts b/tests/routing-matrix.test.ts new file mode 100644 index 0000000..fb0c632 --- /dev/null +++ b/tests/routing-matrix.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from 'vitest'; +import { + L9LLMRouter, + UnsupportedCapabilityCombinationError, + requiresSearchProvider, + resolveRoute, +} from '../src/index.js'; +import { resolvePerplexityConfig } from '../src/matrices/perplexity-matrix.js'; +import { buildRequestBody } from '../src/providers/perplexity.js'; +import { + GeneralModel, + Provider, + TaskComplexity, + TaskType, + type GeneralModelConfig, + type LLMResponse, + type PerplexityConfig, + type TaskDescriptor, + type VisionConfig, +} from '../src/types.js'; + +const IMAGES = ['https://cdn.example.com/screenshot.png']; + +const base = (over: Partial): TaskDescriptor => ({ + type: TaskType.CLASSIFICATION, + complexity: TaskComplexity.MEDIUM, + clientId: 'client', + ...over, +}); + +describe('routing matrix — explicit requiresSearch is authoritative', () => { + const matrix: Array<{ name: string; task: TaskDescriptor; expected: Provider | 'conflict' }> = [ + { name: 'A. STRATEGIC_REASONING + false → NON_SEARCH', task: base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), expected: Provider.OPENROUTER }, + { name: 'B. STRATEGIC_REASONING + true → SEARCH', task: base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true }), expected: Provider.PERPLEXITY }, + { name: 'C. STRATEGIC_REASONING + undefined → NON_SEARCH', task: base({ type: TaskType.STRATEGIC_REASONING }), expected: Provider.OPENROUTER }, + { name: 'D. COMPETITOR_RESEARCH + true → SEARCH', task: base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: true }), expected: Provider.PERPLEXITY }, + { name: 'E. COMPETITOR_RESEARCH + false → NON_SEARCH', task: base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false }), expected: Provider.OPENROUTER }, + { name: 'F. COMPETITOR_RESEARCH + undefined → SEARCH', task: base({ type: TaskType.COMPETITOR_RESEARCH }), expected: Provider.PERPLEXITY }, + { name: 'G. FACT_VERIFICATION + false → NON_SEARCH', task: base({ type: TaskType.FACT_VERIFICATION, requiresSearch: false }), expected: Provider.OPENROUTER }, + { name: 'H. FACT_VERIFICATION + undefined → SEARCH', task: base({ type: TaskType.FACT_VERIFICATION }), expected: Provider.PERPLEXITY }, + { name: 'I. CONTENT_GENERATION + false → NON_SEARCH', task: base({ type: TaskType.CONTENT_GENERATION, requiresSearch: false }), expected: Provider.OPENROUTER }, + { name: 'J. CONTENT_GENERATION + true → SEARCH', task: base({ type: TaskType.CONTENT_GENERATION, requiresSearch: true }), expected: Provider.PERPLEXITY }, + { name: 'K. SCREENSHOT_ANALYSIS + images + false → VISION', task: base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: false }), expected: Provider.OPENROUTER }, + { name: 'L. SCREENSHOT_ANALYSIS + images + undefined → VISION', task: base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES }), expected: Provider.OPENROUTER }, + { name: 'M. SCREENSHOT_ANALYSIS + images + true → fail closed', task: base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: true }), expected: 'conflict' }, + ]; + + it.each(matrix)('$name', ({ task, expected }) => { + if (expected === 'conflict') { + expect(() => resolveRoute(task)).toThrow(UnsupportedCapabilityCombinationError); + return; + } + expect(resolveRoute(task).provider).toBe(expected); + }); +}); + +describe('routing audit evidence', () => { + it('explicit requiresSearch=false → searchRequired=false + EXPLICIT', () => { + const decision = resolveRoute(base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false })); + expect(decision).toMatchObject({ searchRequired: false, searchPolicySource: 'EXPLICIT', provider: Provider.OPENROUTER }); + }); + + it('explicit requiresSearch=true → searchRequired=true + EXPLICIT', () => { + const decision = resolveRoute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true })); + expect(decision).toMatchObject({ searchRequired: true, searchPolicySource: 'EXPLICIT', provider: Provider.PERPLEXITY }); + }); + + it('omitted flag on a default-search task → TASK_DEFAULT', () => { + const decision = resolveRoute(base({ type: TaskType.MARKET_RESEARCH })); + expect(decision).toMatchObject({ searchRequired: true, searchPolicySource: 'TASK_DEFAULT' }); + }); + + it('omitted flag on a general task → TASK_DEFAULT', () => { + const decision = resolveRoute(base({ type: TaskType.STRATEGIC_REASONING })); + expect(decision).toMatchObject({ searchRequired: false, searchPolicySource: 'TASK_DEFAULT' }); + }); + + it('call log carries the resolved search policy and dispatched provider/model', async () => { + let selectedModel: GeneralModel | undefined; + const capturingOpenRouter = { + ...fakeOpenRouter, + completeWithFallback: async (config: GeneralModelConfig) => { + selectedModel = config.model; + return { ...response, model: config.model }; + }, + }; + const router = new L9LLMRouter({ perplexityApiKey: 'p', openrouterApiKey: 'o' }, { openrouterClient: capturingOpenRouter, perplexityClient: fakePerplexity }); + router.initClient('client'); + const result = await router.execute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), 's', 'u'); + const entry = router.getCallLog()[0]; + expect(entry).toMatchObject({ + searchRequired: false, + searchPolicySource: 'EXPLICIT', + provider: Provider.OPENROUTER, + model: selectedModel, + actualCost: result.cost, + latencyMs: result.latencyMs, + }); + }); +}); + +describe('provider dispatch — explicit search policy selects the right client', () => { + it('requiresSearch=true calls the search client and never the general client', async () => { + const { perplexityCalls, openrouterCalls, router } = makeCountingRouter(); + router.initClient('client'); + await router.execute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true }), 's', 'u'); + expect(perplexityCalls()).toBe(1); + expect(openrouterCalls()).toBe(0); + }); + + it('requiresSearch=false on a default-search TaskType calls the general client and never the search client', async () => { + const { perplexityCalls, openrouterCalls, router } = makeCountingRouter(); + router.initClient('client'); + await router.execute(base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false }), 's', 'u'); + expect(openrouterCalls()).toBe(1); + expect(perplexityCalls()).toBe(0); + }); + + it('vision tasks dispatch through the vision path', async () => { + const { perplexityCalls, visionCalls, router } = makeCountingRouter(); + router.initClient('client'); + await router.execute(base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES }), 's', 'u'); + expect(visionCalls()).toBe(1); + expect(perplexityCalls()).toBe(0); + }); + + it('unsupported search+vision fails closed with no provider dispatch and no budget leak', async () => { + const { perplexityCalls, openrouterCalls, router } = makeCountingRouter(); + router.initClient('client'); + await expect(router.execute(base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: true }), 's', 'u')) + .rejects.toBeInstanceOf(UnsupportedCapabilityCombinationError); + expect(perplexityCalls()).toBe(0); + expect(openrouterCalls()).toBe(0); + expect(router.getClientBudgetReport('client')).toMatchObject({ reservedSpend: 0, activeReservations: 0 }); + expect(router.getCircuitState(Provider.PERPLEXITY).failureCount).toBe(0); + expect(router.getCircuitState(Provider.OPENROUTER).failureCount).toBe(0); + }); +}); + +describe('consensus is an execution modifier, not search-policy authority', () => { + it('consensus=true on a non-search route neither reroutes nor errors', async () => { + const { perplexityCalls, router } = makeCountingRouter(); + router.initClient('client'); + const result = await router.execute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), 's', 'u', { consensus: true }); + expect(result.provider).toBe(Provider.OPENROUTER); + expect(perplexityCalls()).toBe(0); + expect(router.getCallLog()[0]).toMatchObject({ searchRequired: false, searchPolicySource: 'EXPLICIT' }); + }); +}); + +describe('Perplexity config agrees with the route decision', () => { + it('never disables search for any router-selected Perplexity route', () => { + for (const type of Object.values(TaskType)) { + for (const requiresSearch of [true, undefined] as const) { + const task = base({ type, requiresSearch }); + if (!requiresSearchProvider(task)) continue; + expect(resolvePerplexityConfig(task).disableSearch, `${type} requiresSearch=${requiresSearch}`).toBe(false); + } + } + }); + + it('a resolved search config produces a request body with web search enabled', () => { + const config = resolvePerplexityConfig(base({ type: TaskType.MARKET_RESEARCH })); + const body = buildRequestBody(config, []); + expect(body.web_search_options).toBeDefined(); + }); +}); + +const response: LLMResponse = { content: 'ok', model: GeneralModel.GPT4O_MINI, provider: Provider.OPENROUTER, inputTokens: 1, outputTokens: 1, totalTokens: 2, cost: 0.1, latencyMs: 5, cached: false }; +const fakeOpenRouter = { + complete: async (_config: GeneralModelConfig) => response, + completeWithVision: async (_config: VisionConfig) => response, + completeWithFallback: async (_config: GeneralModelConfig) => response, +}; +const fakePerplexity = { + complete: async (_config: PerplexityConfig) => ({ ...response, provider: Provider.PERPLEXITY }), + completeWithConsensus: async (_config: PerplexityConfig) => ({ + best: { ...response, provider: Provider.PERPLEXITY }, + all: [{ ...response, provider: Provider.PERPLEXITY }], + consensusScore: 1, + aggregate: { inputTokens: 1, outputTokens: 1, totalTokens: 2, cost: 0.1, latencyMs: 5, citations: [] }, + }), +}; + +function makeCountingRouter() { + let perplexityCalls = 0; + let openrouterCalls = 0; + let visionCalls = 0; + const countingOpenRouter = { + complete: async (_config: GeneralModelConfig) => { openrouterCalls += 1; return response; }, + completeWithVision: async (_config: VisionConfig) => { visionCalls += 1; return response; }, + completeWithFallback: async (_config: GeneralModelConfig) => { openrouterCalls += 1; return response; }, + }; + const countingPerplexity = { + ...fakePerplexity, + complete: async (_config: PerplexityConfig) => { perplexityCalls += 1; return { ...response, provider: Provider.PERPLEXITY }; }, + }; + const router = new L9LLMRouter( + { perplexityApiKey: 'p', openrouterApiKey: 'o' }, + { openrouterClient: countingOpenRouter, perplexityClient: countingPerplexity }, + ); + return { perplexityCalls: () => perplexityCalls, openrouterCalls: () => openrouterCalls, visionCalls: () => visionCalls, router }; +}