Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import {
import { CircuitBreaker, CircuitOpenError, type CircuitPermit } from './circuit-breaker.js';
import { resolveGeneralConfig, getFallbackChain } from './matrices/general-matrix.js';
import { resolvePerplexityConfig } from './matrices/perplexity-matrix.js';
import { requiresSearchProvider } from './matrices/search-policy.js';
import { assertSearchVisionCompatible, requiresSearchProvider } from './matrices/search-policy.js';
import { classifyProviderError, isCircuitFailure } from './provider-errors.js';
import { OpenRouterClient, validateImageUrl, type OpenRouterClientLike } from './providers/openrouter.js';
import { PerplexityClient, type PerplexityClientLike } from './providers/perplexity.js';
Expand All @@ -25,6 +25,7 @@ import {
type RouterConfig,
type RoutingDecision,
type RoutingResolution,
type SearchPolicySource,
type TaskDescriptor,
} from './types.js';
import { generateFullSiteQAPlan, resolveVisionConfig, VIEWPORTS, type FullSiteQAConfig, type VisualQATask } from './vision/index.js';
Expand All @@ -41,16 +42,19 @@ export interface RouterDependencies {
}

export function resolveRoute(task: TaskDescriptor): RoutingResolution {
if (requiresSearchProvider(task)) {
const searchRequired = requiresSearchProvider(task);
const searchPolicySource: SearchPolicySource = typeof task.requiresSearch === 'boolean' ? 'EXPLICIT' : 'TASK_DEFAULT';
assertSearchVisionCompatible(task, searchRequired);
if (searchRequired) {
const config = resolvePerplexityConfig(task);
return { taskType: task.type, complexity: task.complexity, provider: Provider.PERPLEXITY, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason };
return { taskType: task.type, complexity: task.complexity, provider: Provider.PERPLEXITY, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason, searchRequired, searchPolicySource };
}
if (VISION_TASKS.has(task.type)) {
const config = resolveVisionConfig(task.type as TaskType.VISUAL_QA | TaskType.SCREENSHOT_ANALYSIS | TaskType.LAYOUT_VALIDATION, task.complexity, task.images?.length ?? 1);
return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason };
return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason, searchRequired: false, searchPolicySource };
}
const config = resolveGeneralConfig(task);
return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason };
return { taskType: task.type, complexity: task.complexity, provider: Provider.OPENROUTER, model: config.model, estimatedCost: config.estimatedCostPerCall, reason: config.resolutionReason, searchRequired: false, searchPolicySource };
}

export function getDowngradedModel(
Expand Down Expand Up @@ -247,7 +251,7 @@ export { ProviderRequestError } from './provider-errors.js';
export { TaskValidationError, RouterConfigValidationError } from './schemas.js';
export { UnsafeImageUrlError, InvalidBaseUrlError, DEFAULT_OPENROUTER_BASE_URL, resolveOpenRouterBaseUrl } from './providers/openrouter.js';
export { VIEWPORTS } from './vision/index.js';
export { isSearchTask, requiresSearchProvider } from './matrices/search-policy.js';
export { isSearchTask, requiresSearchProvider, UnsupportedCapabilityCombinationError, assertSearchVisionCompatible } from './matrices/search-policy.js';

export { hydrateRouterPrompt } from './memory.js';
export type { RouterMemoryConfig } from './memory.js';
6 changes: 5 additions & 1 deletion src/matrices/perplexity-matrix.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

// Re-exported for backward compatibility: `isSearchTask` historically lived in
// this module. Its canonical home is now ./search-policy.ts.
export { isSearchTask };

Check warning on line 17 in src/matrices/perplexity-matrix.ts

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use `export…from` to re-export `isSearchTask`.

See more on https://sonarcloud.io/project/issues?id=Quantum-L9_LLM-Router&issues=AaAP2R6qfDLNwDcNczqu&open=AaAP2R6qfDLNwDcNczqu&pullRequest=55

function selectSonarModel(complexity: TaskComplexity, rank: number): SonarModel {
if (complexity === TaskComplexity.CRITICAL) return SonarModel.SONAR_DEEP_RESEARCH;
Expand Down Expand Up @@ -57,7 +57,11 @@
domainFilter: task.domainFilter ?? [],
variations,
reasoningEffort: selectReasoningEffort(model, task.complexity),
disableSearch: task.requiresSearch === false && !isSearchTask(task.type),
// A Perplexity route is only selected when requiresSearchProvider(task) is
// true (search required by explicit flag or TaskType default), so a resolved
// router config never disables search. The field stays on the config
// contract for direct-provider callers; the router always sets false.
disableSearch: false,
estimatedCostPerCall,
resolutionReason: `Task[${task.type}] complexity[${task.complexity}] uses ${model}`,
};
Expand Down
30 changes: 30 additions & 0 deletions src/matrices/search-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,33 @@ export function requiresSearchProvider(task: TaskDescriptor): boolean {
}
return isSearchTask(task.type);
}

/**
* Fail-closed guard for capability combinations no current provider supports.
*
* The Perplexity client is text-only, so a search-required route cannot consume
* images. Rather than silently dropping either requested capability (dropping
* the images, or routing vision while pretending search happened), resolution
* refuses the combination until a multimodal-search provider contract exists.
*/
export class UnsupportedCapabilityCombinationError extends Error {
public readonly code = 'SEARCH_VISION_COMBINATION_UNSUPPORTED' as const;
constructor(public readonly taskType: TaskType, public readonly capabilities: readonly string[]) {
super(`Task type "${taskType}" requires an unsupported capability combination: ${capabilities.join(' + ')}`);
this.name = 'UnsupportedCapabilityCombinationError';
}
toJSON(): Record<string, unknown> {
return { name: this.name, code: this.code, taskType: this.taskType, capabilities: this.capabilities, message: this.message };
}
}

/**
* Throws {@link UnsupportedCapabilityCombinationError} when a search-required
* route would also have to consume images. No silent capability loss: neither
* the images nor the search request is dropped behind the caller's back.
*/
export function assertSearchVisionCompatible(task: TaskDescriptor, searchRequired: boolean): void {
if (searchRequired && (task.images?.length ?? 0) > 0) {
throw new UnsupportedCapabilityCombinationError(task.type, ['search', 'vision']);
}
}
1 change: 1 addition & 0 deletions src/provider-errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const LOCAL_ERROR_NAMES = new Set([
'TaskValidationError',
'RouterConfigValidationError',
'UnsafeImageUrlError',
'UnsupportedCapabilityCombinationError',
'BudgetExhaustedError',
'CircuitOpenError',
'AbortError',
Expand Down
6 changes: 6 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -183,13 +183,19 @@ export interface RouterConfig {
providerMaxRetries?: 0;
}

export type SearchPolicySource = 'EXPLICIT' | 'TASK_DEFAULT';

export interface RoutingResolution {
taskType: TaskType;
complexity: TaskComplexity;
provider: Provider;
model: GeneralModel | SonarModel;
estimatedCost: number;
reason: string;
/** Whether the resolved route carries web-search capability (explicit flag or task-type default). */
searchRequired: boolean;
/** Where `searchRequired` came from: an explicit `TaskDescriptor.requiresSearch` boolean, or the legacy per-TaskType default. */
searchPolicySource: SearchPolicySource;
}

export interface RoutingDecision extends RoutingResolution {
Expand Down
203 changes: 203 additions & 0 deletions tests/routing-matrix.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
import { describe, expect, it } from 'vitest';
import {
L9LLMRouter,
UnsupportedCapabilityCombinationError,
requiresSearchProvider,
resolveRoute,
} from '../src/index.js';
import { resolvePerplexityConfig } from '../src/matrices/perplexity-matrix.js';
import { buildRequestBody } from '../src/providers/perplexity.js';
import {
GeneralModel,
Provider,
TaskComplexity,
TaskType,
type GeneralModelConfig,
type LLMResponse,
type PerplexityConfig,
type TaskDescriptor,
type VisionConfig,
} from '../src/types.js';

const IMAGES = ['https://cdn.example.com/screenshot.png'];

const base = (over: Partial<TaskDescriptor>): TaskDescriptor => ({
type: TaskType.CLASSIFICATION,
complexity: TaskComplexity.MEDIUM,
clientId: 'client',
...over,
});

describe('routing matrix — explicit requiresSearch is authoritative', () => {
const matrix: Array<{ name: string; task: TaskDescriptor; expected: Provider | 'conflict' }> = [
{ name: 'A. STRATEGIC_REASONING + false → NON_SEARCH', task: base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), expected: Provider.OPENROUTER },
{ name: 'B. STRATEGIC_REASONING + true → SEARCH', task: base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true }), expected: Provider.PERPLEXITY },
{ name: 'C. STRATEGIC_REASONING + undefined → NON_SEARCH', task: base({ type: TaskType.STRATEGIC_REASONING }), expected: Provider.OPENROUTER },
{ name: 'D. COMPETITOR_RESEARCH + true → SEARCH', task: base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: true }), expected: Provider.PERPLEXITY },
{ name: 'E. COMPETITOR_RESEARCH + false → NON_SEARCH', task: base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false }), expected: Provider.OPENROUTER },
{ name: 'F. COMPETITOR_RESEARCH + undefined → SEARCH', task: base({ type: TaskType.COMPETITOR_RESEARCH }), expected: Provider.PERPLEXITY },
{ name: 'G. FACT_VERIFICATION + false → NON_SEARCH', task: base({ type: TaskType.FACT_VERIFICATION, requiresSearch: false }), expected: Provider.OPENROUTER },
{ name: 'H. FACT_VERIFICATION + undefined → SEARCH', task: base({ type: TaskType.FACT_VERIFICATION }), expected: Provider.PERPLEXITY },
{ name: 'I. CONTENT_GENERATION + false → NON_SEARCH', task: base({ type: TaskType.CONTENT_GENERATION, requiresSearch: false }), expected: Provider.OPENROUTER },
{ name: 'J. CONTENT_GENERATION + true → SEARCH', task: base({ type: TaskType.CONTENT_GENERATION, requiresSearch: true }), expected: Provider.PERPLEXITY },
{ name: 'K. SCREENSHOT_ANALYSIS + images + false → VISION', task: base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: false }), expected: Provider.OPENROUTER },
{ name: 'L. SCREENSHOT_ANALYSIS + images + undefined → VISION', task: base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES }), expected: Provider.OPENROUTER },
{ name: 'M. SCREENSHOT_ANALYSIS + images + true → fail closed', task: base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: true }), expected: 'conflict' },
];

it.each(matrix)('$name', ({ task, expected }) => {
if (expected === 'conflict') {
expect(() => resolveRoute(task)).toThrow(UnsupportedCapabilityCombinationError);
return;
}
expect(resolveRoute(task).provider).toBe(expected);
});
});

describe('routing audit evidence', () => {
it('explicit requiresSearch=false → searchRequired=false + EXPLICIT', () => {
const decision = resolveRoute(base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false }));
expect(decision).toMatchObject({ searchRequired: false, searchPolicySource: 'EXPLICIT', provider: Provider.OPENROUTER });
});

it('explicit requiresSearch=true → searchRequired=true + EXPLICIT', () => {
const decision = resolveRoute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true }));
expect(decision).toMatchObject({ searchRequired: true, searchPolicySource: 'EXPLICIT', provider: Provider.PERPLEXITY });
});

it('omitted flag on a default-search task → TASK_DEFAULT', () => {
const decision = resolveRoute(base({ type: TaskType.MARKET_RESEARCH }));
expect(decision).toMatchObject({ searchRequired: true, searchPolicySource: 'TASK_DEFAULT' });
});

it('omitted flag on a general task → TASK_DEFAULT', () => {
const decision = resolveRoute(base({ type: TaskType.STRATEGIC_REASONING }));
expect(decision).toMatchObject({ searchRequired: false, searchPolicySource: 'TASK_DEFAULT' });
});

it('call log carries the resolved search policy and dispatched provider/model', async () => {
let selectedModel: GeneralModel | undefined;
const capturingOpenRouter = {
...fakeOpenRouter,
completeWithFallback: async (config: GeneralModelConfig) => {
selectedModel = config.model;
return { ...response, model: config.model };
},
};
const router = new L9LLMRouter({ perplexityApiKey: 'p', openrouterApiKey: 'o' }, { openrouterClient: capturingOpenRouter, perplexityClient: fakePerplexity });
router.initClient('client');
const result = await router.execute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), 's', 'u');
const entry = router.getCallLog()[0];
expect(entry).toMatchObject({
searchRequired: false,
searchPolicySource: 'EXPLICIT',
provider: Provider.OPENROUTER,
model: selectedModel,
actualCost: result.cost,
latencyMs: result.latencyMs,
});
});
});

describe('provider dispatch — explicit search policy selects the right client', () => {
it('requiresSearch=true calls the search client and never the general client', async () => {
const { perplexityCalls, openrouterCalls, router } = makeCountingRouter();
router.initClient('client');
await router.execute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: true }), 's', 'u');
expect(perplexityCalls()).toBe(1);
expect(openrouterCalls()).toBe(0);
});

it('requiresSearch=false on a default-search TaskType calls the general client and never the search client', async () => {
const { perplexityCalls, openrouterCalls, router } = makeCountingRouter();
router.initClient('client');
await router.execute(base({ type: TaskType.COMPETITOR_RESEARCH, requiresSearch: false }), 's', 'u');
expect(openrouterCalls()).toBe(1);
expect(perplexityCalls()).toBe(0);
});

it('vision tasks dispatch through the vision path', async () => {
const { perplexityCalls, visionCalls, router } = makeCountingRouter();
router.initClient('client');
await router.execute(base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES }), 's', 'u');
expect(visionCalls()).toBe(1);
expect(perplexityCalls()).toBe(0);
});

it('unsupported search+vision fails closed with no provider dispatch and no budget leak', async () => {
const { perplexityCalls, openrouterCalls, router } = makeCountingRouter();
router.initClient('client');
await expect(router.execute(base({ type: TaskType.SCREENSHOT_ANALYSIS, images: IMAGES, requiresSearch: true }), 's', 'u'))
.rejects.toBeInstanceOf(UnsupportedCapabilityCombinationError);
expect(perplexityCalls()).toBe(0);
expect(openrouterCalls()).toBe(0);
expect(router.getClientBudgetReport('client')).toMatchObject({ reservedSpend: 0, activeReservations: 0 });
expect(router.getCircuitState(Provider.PERPLEXITY).failureCount).toBe(0);
expect(router.getCircuitState(Provider.OPENROUTER).failureCount).toBe(0);
});
});

describe('consensus is an execution modifier, not search-policy authority', () => {
it('consensus=true on a non-search route neither reroutes nor errors', async () => {
const { perplexityCalls, router } = makeCountingRouter();
router.initClient('client');
const result = await router.execute(base({ type: TaskType.STRATEGIC_REASONING, requiresSearch: false }), 's', 'u', { consensus: true });
expect(result.provider).toBe(Provider.OPENROUTER);
expect(perplexityCalls()).toBe(0);
expect(router.getCallLog()[0]).toMatchObject({ searchRequired: false, searchPolicySource: 'EXPLICIT' });
});
});

describe('Perplexity config agrees with the route decision', () => {
it('never disables search for any router-selected Perplexity route', () => {
for (const type of Object.values(TaskType)) {
for (const requiresSearch of [true, undefined] as const) {
const task = base({ type, requiresSearch });
if (!requiresSearchProvider(task)) continue;
expect(resolvePerplexityConfig(task).disableSearch, `${type} requiresSearch=${requiresSearch}`).toBe(false);
}
}
});

it('a resolved search config produces a request body with web search enabled', () => {
const config = resolvePerplexityConfig(base({ type: TaskType.MARKET_RESEARCH }));
const body = buildRequestBody(config, []);
expect(body.web_search_options).toBeDefined();
});
});

const response: LLMResponse = { content: 'ok', model: GeneralModel.GPT4O_MINI, provider: Provider.OPENROUTER, inputTokens: 1, outputTokens: 1, totalTokens: 2, cost: 0.1, latencyMs: 5, cached: false };
const fakeOpenRouter = {
complete: async (_config: GeneralModelConfig) => response,
completeWithVision: async (_config: VisionConfig) => response,
completeWithFallback: async (_config: GeneralModelConfig) => response,
};
const fakePerplexity = {
complete: async (_config: PerplexityConfig) => ({ ...response, provider: Provider.PERPLEXITY }),
completeWithConsensus: async (_config: PerplexityConfig) => ({
best: { ...response, provider: Provider.PERPLEXITY },
all: [{ ...response, provider: Provider.PERPLEXITY }],
consensusScore: 1,
aggregate: { inputTokens: 1, outputTokens: 1, totalTokens: 2, cost: 0.1, latencyMs: 5, citations: [] },
}),
};

function makeCountingRouter() {
let perplexityCalls = 0;
let openrouterCalls = 0;
let visionCalls = 0;
const countingOpenRouter = {
complete: async (_config: GeneralModelConfig) => { openrouterCalls += 1; return response; },
completeWithVision: async (_config: VisionConfig) => { visionCalls += 1; return response; },
completeWithFallback: async (_config: GeneralModelConfig) => { openrouterCalls += 1; return response; },
};
const countingPerplexity = {
...fakePerplexity,
complete: async (_config: PerplexityConfig) => { perplexityCalls += 1; return { ...response, provider: Provider.PERPLEXITY }; },
};
const router = new L9LLMRouter(
{ perplexityApiKey: 'p', openrouterApiKey: 'o' },
{ openrouterClient: countingOpenRouter, perplexityClient: countingPerplexity },
);
return { perplexityCalls: () => perplexityCalls, openrouterCalls: () => openrouterCalls, visionCalls: () => visionCalls, router };
}
Loading