From 696a2ddffe387733814cbd9eb09f112b47c30dc9 Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:01:05 +0800 Subject: [PATCH] feat: mark a provider model retired when its provider stops offering it (#852) --- changelog.d/852-model-retired.added.md | 1 + docs/architecture.md | 12 ++ docs/quota.md | 6 + src/app.ts | 10 ++ src/db.ts | 25 +++- src/gateway/gateway.ts | 10 +- src/gateway/modelFallback.ts | 10 +- src/lib/modelRun.ts | 3 + src/views/pages.ts | 26 +++- test/db.test.ts | 6 +- test/modelFallback.test.ts | 1 + test/modelRetire.test.ts | 173 +++++++++++++++++++++++++ test/notificationOutbox.test.ts | 4 +- test/scheduleLeases.test.ts | 4 +- test/scheduleOccurrences.test.ts | 4 +- test/scheduleSchema.test.ts | 4 +- 16 files changed, 278 insertions(+), 21 deletions(-) create mode 100644 changelog.d/852-model-retired.added.md create mode 100644 test/modelRetire.test.ts diff --git a/changelog.d/852-model-retired.added.md b/changelog.d/852-model-retired.added.md new file mode 100644 index 00000000..06ad71ae --- /dev/null +++ b/changelog.d/852-model-retired.added.md @@ -0,0 +1 @@ +A provider model its provider has stopped serving is recorded as retired with the date: Settings and the repository page say so, the job's model line names the reason, and an apply that would restart a workspace onto that model is refused with its name. diff --git a/docs/architecture.md b/docs/architecture.md index 0e376d44..3ab37794 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1792,6 +1792,18 @@ settings form disables the checkbox of a provider whose allowed flag is off. existing repository keeps the unrestricted behaviour until an owner sets a policy. +A provider can stop offering a model it used to serve. The gateway already +treated a 404 whose body names the model as retryable so a job could finish on +another model; it now also writes `provider_models.retired_at` — the first date +that 404 was seen, never moved by a later one — and marks the attempt in the +job's `model_json` with `retired: true`. Settings shows the model with its +retirement date, the repository page marks a retired default and any per-bot +picker that points at it, and the job's model line names the reason beside the +status. An apply whose selection still holds a retired model is refused with +that model's name, so a restart cannot move a workspace onto a model its provider +no longer serves; saving the selection itself still works. Only a 404 matching the +retired-model pattern writes the column — a 429, a 5xx and any other 404 never do. + ### Per-job cost, evidence and metrics The bridge's model proxy normally uses the per-repository instance token. While diff --git a/docs/quota.md b/docs/quota.md index 91c5be3d..3ea90221 100644 --- a/docs/quota.md +++ b/docs/quota.md @@ -73,6 +73,12 @@ writes one of those rows rather than none: the provider's `x-litellm-response-co figure as a known USD row when the response carried one, and a `cost_known = 0` row with reason `no_usage`, its audit entry and its job event when it did not. +An attempt that never reached a model is not billed: a `404` whose body says the +model does not exist writes no ledger row, so a model its provider has retired +costs nothing, and the fallback to the next selected model is priced on its own +answer. The retirement itself is recorded on the provider model +(`provider_models.retired_at`, #852), not in the ledger. + For streaming requests Fledge adds `stream_options.include_usage = true` and parses each SSE `data:` line. It keeps the last `usage` object, including nested `completion_tokens_details` / `prompt_tokens_details`, and meters after the diff --git a/src/app.ts b/src/app.ts index 15cf969a..e034b726 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4378,6 +4378,16 @@ export async function buildApp(options: BuildOptions): Promise { app.post('/repos/:id/models/apply', async (request, reply) => { const { viewer, repo } = ownedRepo(request, String((request.params as Record).id), true, true) + // #852: a retired model is still in the selection, so applying would restart + // the workspace onto a model its provider no longer serves. Refused by name + // rather than deleted: the maintainer picks a live model and nothing is lost. + const retiredSelection = store.listRepoModelSelection(repo.id).find((model) => model.retired_at !== null && model.retired_at !== undefined) + if (retiredSelection !== undefined) { + const t = translator(request.locale ?? 'zh') + const on = String(retiredSelection.retired_at).slice(0, 10) + const message = t(`模型“${retiredSelection.label}”已于 ${on} 被提供方下架,请先选择一个可用的模型再应用。`, `The model "${retiredSelection.label}" was retired by its provider on ${on}. Choose a live model before applying.`) + return reply.redirect(`/repos/${repo.id}?error=${encodeURIComponent(message)}#repo-settings`) + } try { const result = await instances.applyModels(repo.id) if (!result.ok) return reply.redirect(`/repos/${repo.id}?error=${encodeURIComponent(result.error ?? '模型应用失败。')}#repo-settings`) diff --git a/src/db.ts b/src/db.ts index 517879dc..1a3f2f7b 100644 --- a/src/db.ts +++ b/src/db.ts @@ -62,7 +62,7 @@ const { DatabaseSync } = require('node:sqlite') as { DatabaseSync: new (path: st // restore check (#652) has to prove a fixture written by an older build still // migrates to exactly this number, and a literal in two places would let the // check and the migration disagree. -export const SCHEMA_VERSION = 44 +export const SCHEMA_VERSION = 45 export type UserStatus = 'onboarding' | 'pending' | 'approved' | 'rejected' export type UserRole = 'user' | 'admin' | 'superadmin' @@ -383,6 +383,8 @@ export interface ProviderModelRow { provider_id: string model: string label: string + /** #852: the first day a provider answered "this model does not exist"; null while live. */ + retired_at: string | null created_at: string } @@ -1493,6 +1495,7 @@ CREATE TABLE IF NOT EXISTS provider_models ( provider_id TEXT NOT NULL REFERENCES model_providers(id) ON DELETE CASCADE, model TEXT NOT NULL, label TEXT NOT NULL, + retired_at TEXT, created_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS provider_models_provider_idx ON provider_models(provider_id, created_at ASC); @@ -2416,6 +2419,7 @@ export class Store { this.migrateColumns() this.migrateRepoColumns() this.migrateProviderColumns() + this.migrateProviderModelColumns() this.migrateWebhookColumns() this.migrateArtifactColumns() this.migrateManagementColumns() @@ -2839,6 +2843,14 @@ export class Store { this.run("UPDATE jobs SET base_branch = (SELECT upstream_default_branch FROM repos WHERE repos.id = jobs.repo_id) WHERE base_branch IS NULL") } + // #852: a provider can stop offering a model it used to serve, and the row must + // carry that fact so the console can say it. Additive: a row written before the + // column keeps NULL, which is exactly the "live" it was always shown as. + private migrateProviderModelColumns(): void { + const columns = new Set(this.all<{ name: string }>('PRAGMA table_info(provider_models)').map((row) => row.name)) + if (!columns.has('retired_at')) this.run('ALTER TABLE provider_models ADD COLUMN retired_at TEXT') + } + private migrateProviderColumns(): void { const columns = new Set(this.all<{ name: string }>('PRAGMA table_info(model_providers)').map((row) => row.name)) const additions: Array<[string, string]> = [ @@ -4870,6 +4882,17 @@ export class Store { ) } + // #852: the first date wins. A second 404 for the same model must not move it, + // so the write is decided by the WHERE clause rather than by the caller. + markProviderModelRetired(modelId: string, at: string): boolean { + return this.transaction(() => { + const live = this.get<{ id: string }>('SELECT id FROM provider_models WHERE id = ? AND retired_at IS NULL', modelId) + if (live === undefined) return false + this.run('UPDATE provider_models SET retired_at = ? WHERE id = ?', at, modelId) + return true + }) + } + createProviderModel(input: { id: string; providerId: string; model: string; label: string }): ProviderModelRow { this.run('INSERT INTO provider_models (id, provider_id, model, label, created_at) VALUES (?, ?, ?, ?, ?)', input.id, input.providerId, input.model, input.label, nowIso()) return this.get('SELECT * FROM provider_models WHERE id = ?', input.id)! diff --git a/src/gateway/gateway.ts b/src/gateway/gateway.ts index 75103976..631f4c6c 100644 --- a/src/gateway/gateway.ts +++ b/src/gateway/gateway.ts @@ -8,7 +8,8 @@ import { log } from '../lib/log.js' import { decryptSecret } from '../lib/secrets.js' import { headerCost, quotaView } from './quota.js' import { fallbackPriceTable, parsePriceTable, requestCost, type PriceTable } from '../metrics/cost.js' -import { modelOrder, retryableModelFailure } from './modelFallback.js' +import { modelOrder, retiredModelFailure, retryableModelFailure } from './modelFallback.js' +import { nowIso } from '../lib/time.js' import { parseProviderAllowlist, providerPolicyRefusal } from './modelUrl.js' import { parseModelRun } from '../lib/modelRun.js' import type { ModelAttempt, ModelFallback } from '../lib/modelRun.js' @@ -318,7 +319,12 @@ export function registerGateway(app: FastifyInstance, deps: GatewayDeps): void { } const text = await readAnswer(answer.res) const retryable = retryableModelFailure(answer.status, text) - attempts.push({ ...attemptName, status: answer.status, retryable }) + // #852: a retired model is the one 404 that is not about the URL. The row is + // written once with the first date, so the console can say when it happened; + // a 429, a 5xx or any other 404 never writes it. + const retired = retiredModelFailure(answer.status, text) + if (retired && candidate !== undefined) store.markProviderModelRetired(candidate.id, nowIso()) + attempts.push({ ...attemptName, status: answer.status, retryable, ...(retired ? { retired: true } : {}) }) lastFailure = { status: answer.status, text, headers: { ...forwardHeaders(answer.headers), ...quotaHeaders() } } // Auth, an exhausted quota, a payment problem and a policy refusal are the // repository's own answer: a second model must not mask them. diff --git a/src/gateway/modelFallback.ts b/src/gateway/modelFallback.ts index 3e71100f..752dcad4 100644 --- a/src/gateway/modelFallback.ts +++ b/src/gateway/modelFallback.ts @@ -25,6 +25,14 @@ export function modelOrder( const QUOTA_WORDS = /insufficient_quota|quota|billing|payment|budget|额度|余额/i const POLICY_WORDS = /content[_ ]?policy|policy[_ ]?violation|safety|refus/i +// #852: the body that makes a 404 retryable is also the body that says the model +// is gone for good, so the two decisions share one pattern and cannot drift. +const RETIRED_MODEL_WORDS = /model[^\n]{0,60}(not found|not exist|unavailable|decommission|retired)/i + +export function retiredModelFailure(status: number, body: string): boolean { + return status === 404 && RETIRED_MODEL_WORDS.test(body) +} + export function retryableModelFailure(status: number, body: string): boolean { // Auth is the repository's credential, not the model's health: another model // behind the same key is no more likely to be accepted. @@ -35,7 +43,7 @@ export function retryableModelFailure(status: number, body: string): boolean { if (status === 408) return true // A retired model is the one "unavailable" a 404 can mean; any other 404 is a // wrong URL, which another model will not repair. - if (status === 404) return /model[^\n]{0,60}(not found|not exist|unavailable|decommission|retired)/i.test(body) + if (status === 404) return retiredModelFailure(status, body) if (status === 500 || status === 502 || status === 503 || status === 504) return true return false } diff --git a/src/lib/modelRun.ts b/src/lib/modelRun.ts index 38e1439a..7f0f9ca8 100644 --- a/src/lib/modelRun.ts +++ b/src/lib/modelRun.ts @@ -12,6 +12,8 @@ export interface ModelAttempt { label: string status: number retryable: boolean + /** #852: the provider answered that this model does not exist, so it is retired. */ + retired?: boolean } // The first fallback the job made, kept apart from `attempts` because those are @@ -40,6 +42,7 @@ function attempt(value: unknown): ModelAttempt | null { label: typeof row.label === 'string' && row.label !== '' ? row.label : modelId, status, retryable: row.retryable === true, + ...(row.retired === true ? { retired: true } : {}), } } diff --git a/src/views/pages.ts b/src/views/pages.ts index f1e5d90f..82a42ed9 100644 --- a/src/views/pages.ts +++ b/src/views/pages.ts @@ -145,6 +145,17 @@ function commandHidden(fields: { revision: number; idempotencyKey: string } | un return `` } +// #852: a provider can stop offering a model it used to serve, and the row keeps +// the first day it said so. The marker only ever reads that column; a model that +// has answered is never labelled, and nothing here guesses a state. +function retiredOn(model: { retired_at?: string | null }): string { + return model.retired_at === null || model.retired_at === undefined ? '' : String(model.retired_at).slice(0, 10) +} + +function retiredMark(model: { retired_at?: string | null }, t: Translate): string { + return retiredOn(model) === '' ? '' : ` · ${t('已下架', 'retired')}` +} + function infoTip(text: string): string { return `` } @@ -1867,7 +1878,7 @@ export function repoPage(input: { config: Config; locale: Locale; viewer: Viewer
@@ -1879,8 +1890,8 @@ export function repoPage(input: { config: Config; locale: Locale; viewer: Viewer

${escapeHtml(t('区域约束比较提供方标注的区域(不区分大小写);留空表示不限区域。这个策略只作用于当前仓库。', 'The region constraint compares the provider region label case-insensitively; empty allows any region. This policy applies to this repository only.'))}

- - ${models.length === 0 ? '' : `
${([['chief', 'Chief'], ['lead', 'Lead'], ['product', 'Product'], ['test', 'Test'], ['security', 'Security'], ['reviewer', 'Reviewer'], ['dev1', 'Dev-1'], ['dev2', 'Dev-2']] as const).map(([role, label]) => ``).join('')}
+ + ${models.length === 0 ? '' : `
${([['chief', 'Chief'], ['lead', 'Lead'], ['product', 'Product'], ['test', 'Test'], ['security', 'Security'], ['reviewer', 'Reviewer'], ['dev1', 'Dev-1'], ['dev2', 'Dev-2']] as const).map(([role, label]) => ``).join('')}

${escapeHtml(t('每个 bot 可以单独选择模型;不选则跟随默认。保存选择后,空闲时点击“应用模型并重启工作区”才会生效,Fledge 会通过 DSH 把模型分配给对应 bot。', 'Each bot can choose a model; empty follows the default. After saving, use "Apply models and restart workspace" while idle; Fledge assigns the model through DSH.'))}

`} ${roleMissing ? `

${escapeHtml(t('先选择你的角色,审核和自动合并才会生效。', 'Choose your role before review or auto-merge can take effect.'))}

` : !githubCanReview ? `

${escapeHtml(t('你在这个上游没有 admin/maintain 权限,审核角色不可用。', 'You do not have admin/maintain permission on this upstream, so the reviewer role is unavailable.'))}

` : !canMaintain ? `

${escapeHtml(t('developer 不能启用 PR 审核或自动合并;切换为 maintainer 后才可用。', 'A developer cannot enable PR review or auto-merge; switch to maintainer first.'))}

` : ''} @@ -2228,7 +2239,7 @@ export function settingsPage(input: { ${provider.models.length === 0 ? `

${escapeHtml(t('还没有模型。', 'No models yet.'))}

` - : `
    ${provider.models.map((model) => `
  • ${escapeHtml(model.label)} ${escapeHtml(model.model)}
    ${csrf(viewer)}
  • `).join('')}
`} + : `
    ${provider.models.map((model) => `
  • ${escapeHtml(model.label)} ${escapeHtml(model.model)}${retiredOn(model) === '' ? '' : ` ${escapeHtml(t(`提供方已于 ${retiredOn(model)} 下架`, `retired by the provider on ${retiredOn(model)}`))}`}
    ${csrf(viewer)}
  • `).join('')}
`}
${csrf(viewer)}
@@ -3241,13 +3252,16 @@ export function jobPage(input: { // primary was rate-limited finished on another model, so the page must name the // one that replied instead of the one the instance was configured with. const modelRun = parseModelRun(job.model_json) - const triedText = (run: typeof modelRun): string => (run === null ? '' : run.attempts.map((entry) => `${entry.label} (${entry.status})`).join(', ')) + // #852: a retired attempt says why, not only that it failed: the status alone + // cannot tell a wrong URL from a model its provider stopped serving. + const triedText = (run: typeof modelRun): string => (run === null ? '' : run.attempts.map((entry) => `${entry.label} (${entry.status}${entry.retired === true ? `, ${t('已下架', 'retired')}` : ''})`).join(', ')) // #407: a fallback an earlier call made is its own clause. The rule: it is stated // unless the `tried` clause of this same line already names the model it ended // on — only then would two clauses repeat one model. So the skip needs the tried // clause to exist (`attempts` non-empty) and a model to compare against; a last // call with no attempt or no winner keeps the pair visible. A row whose status is // not a real HTTP code states the pair without inventing a number for it. + const retiredAttempt = modelRun === null ? null : modelRun.attempts.find((entry) => entry.retired === true) ?? null const fellBackText = (run: typeof modelRun): string => { if (run === null || run.fallback === null) return '' if (run.attempts.length > 0 && run.model !== null && run.fallback.to.label === run.model.label) return '' @@ -3258,7 +3272,7 @@ export function jobPage(input: { } const modelLine = modelRun === null ? '' - : `

${escapeHtml(t('模型', 'Model'))}: ${escapeHtml(modelRun.model === null ? t('没有完成的调用', 'no completed call') : modelRun.model.label)}${modelRun.attempts.length === 0 ? '' : escapeHtml(` · ${t(`此前尝试 ${triedText(modelRun)}`, `tried ${triedText(modelRun)}`)}`)}${escapeHtml(fellBackText(modelRun))}

` + : `

${escapeHtml(t('模型', 'Model'))}: ${escapeHtml(modelRun.model === null ? t('没有完成的调用', 'no completed call') : modelRun.model.label)}${modelRun.attempts.length === 0 ? '' : escapeHtml(` · ${t(`此前尝试 ${triedText(modelRun)}`, `tried ${triedText(modelRun)}`)}`)}${escapeHtml(fellBackText(modelRun))}

` const providerParts = [ job.provider_label === null ? '' : job.provider_label, job.provider_region === null ? '' : job.provider_region, diff --git a/test/db.test.ts b/test/db.test.ts index 462acb72..ebe7dde7 100644 --- a/test/db.test.ts +++ b/test/db.test.ts @@ -151,9 +151,9 @@ test('the role backfill runs once and future schema versions are refused', () => const reopened = new Store(path) assert.equal(reopened.getRepo(repo.id)?.repo_role, 'maintainer') assert.notEqual(reopened.getRepo(repo.id)?.role_checked_at, null) - // The notification outbox moved the build's schema to 44, so the first - // refused version is 45. - reopened.db.exec('PRAGMA user_version = 45;') + // The retired provider model moved the build's schema to 45, so the first + // refused version is 46. + reopened.db.exec('PRAGMA user_version = 46;') reopened.close() assert.throws(() => new Store(path), /newer than this build/) diff --git a/test/modelFallback.test.ts b/test/modelFallback.test.ts index 4f993c3e..280b2f93 100644 --- a/test/modelFallback.test.ts +++ b/test/modelFallback.test.ts @@ -16,6 +16,7 @@ function row(id: string, isDefault: 0 | 1): RepoProviderModelRow { provider_id: 'prov', model: `model-${id}`, label: `Label ${id}`, + retired_at: null, created_at: '2026-11-12T00:00:00.000Z', provider_label: 'Provider', provider_region: 'eu', diff --git a/test/modelRetire.test.ts b/test/modelRetire.test.ts new file mode 100644 index 00000000..a65bd9cf --- /dev/null +++ b/test/modelRetire.test.ts @@ -0,0 +1,173 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { buildApp } from '../src/app.js' +import { Store } from '../src/db.js' +import { encryptSecret, randomToken, sha256Hex } from '../src/lib/secrets.js' +import { parseModelRun } from '../src/lib/modelRun.js' +import { MockRunner } from '../src/jobs/mock.js' +import { flashOf, json, makeUser, readBody, sessionFor, startFakeServer, testConfig, MemoryMailer } from './helpers.js' + +// #852: a provider that stops offering a model it used to serve answers 404 with +// a "model not found" body. The gateway already treats that as retryable so the +// job can finish elsewhere; what it never did was write the fact down. These +// cases drive the seam with fake providers: the row and the pages must say the +// model is retired, a transient failure must never say it, and an apply must not +// restart a workspace onto a model its provider no longer serves. + +type Fake = (request: import('node:http').IncomingMessage, response: import('node:http').ServerResponse) => Promise + +function chatHit(models: string[], answer: (model: string) => { status: number; payload: unknown }): Fake { + return async (request, response) => { + const body = JSON.parse((await readBody(request)).toString('utf8')) as { model?: string } + const model = String(body.model ?? '') + models.push(model) + const { status, payload } = answer(model) + json(response, status, payload) + } +} + +const OK = { status: 200, payload: { id: 'ok', choices: [{ message: { role: 'assistant', content: 'answered' } }], usage: { total_tokens: 4 } } } +const RETIRED = { status: 404, payload: { error: { code: 'model_not_found', message: 'The model gpt-4o does not exist' } } } + +async function fixture(primaryAnswer: Fake, fallbackAnswer: Fake) { + const primaryHits: string[] = [] + const fallbackHits: string[] = [] + const primary = await startFakeServer(primaryAnswer) + const fallback = await startFakeServer(fallbackAnswer) + const config = testConfig({}) + const store = new Store(':memory:') + const built = await buildApp({ config, store, mailer: new MemoryMailer(), runner: new MockRunner(), startScheduler: false }) + const user = makeUser(store, { login: 'retire-user', status: 'approved' }) + const repo = store.createRepo({ id: 'repo_retire', userId: user.id, owner: 'o', name: 'r', defaultBranch: 'main', tokenEnc: 'x', permission: 'admin', repoRole: 'maintainer', goal: '', autoIterate: false, intervalMinutes: 1440 }) + const providerA = store.createModelProvider({ id: 'prv_a', userId: user.id, label: 'Provider A', baseUrl: `${primary.url}/v1`, keyEnc: encryptSecret('key-a', config.tokenEncryptionKey), allowPrivate: true }) + const providerB = store.createModelProvider({ id: 'prv_b', userId: user.id, label: 'Provider B', baseUrl: `${fallback.url}/v1`, keyEnc: encryptSecret('key-b', config.tokenEncryptionKey), allowPrivate: true }) + const modelA = store.createProviderModel({ id: 'pvm_a', providerId: providerA.id, model: 'gpt-4o', label: 'GPT 4o' }) + const modelB = store.createProviderModel({ id: 'pvm_b', providerId: providerB.id, model: 'backup-model', label: 'Backup Model' }) + store.setRepoModelSelection(repo.id, [modelA.id, modelB.id], modelA.id) + const job = store.createJob({ id: 'job_retire', repoId: repo.id, userId: user.id, trigger: 'manual', goal: 'g', callbackHash: 'cb', platformTokenHash: '' }) + const raw = `flg_${randomToken(32)}` + store.createApiToken({ id: 'tok_retire', userId: user.id, kind: 'job', name: 'job:job_retire', tokenHash: sha256Hex(raw), prefix: raw.slice(0, 8), repoId: repo.id, expiresAt: null }) + const call = () => built.app.inject({ + method: 'POST', + url: '/v1/chat/completions', + headers: { authorization: `Bearer ${raw}`, 'content-type': 'application/json' }, + payload: { model: modelA.id, messages: [{ role: 'user', content: 'hi' }] }, + }) + const close = async () => { + await built.close() + await primary.close() + await fallback.close() + } + return { store, built, user, repo, job, modelA, modelB, providerA, providerB, call, close, primaryHits, fallbackHits } +} + +async function page(built: Awaited>['built'], user: Parameters[1], url: string): Promise { + const { cookie } = sessionFor(built, user) + const response = await built.app.inject({ method: 'GET', url, headers: { cookie } }) + assert.equal(response.statusCode, 200, `${url} answered ${response.statusCode}`) + return response.body +} + +test('a provider 404 marks the model retired, every page says so, and a second 404 does not move the date', async () => { + const created: string[] = [] + const fake = await fixture( + chatHit(created, () => RETIRED), + chatHit(created, () => OK), + ) + try { + const response = await fake.call() + assert.equal(response.statusCode, 200, 'the job still finishes on the repository second model') + + // The attempt carries the reason: the word retired, beside the status. + const run = parseModelRun(fake.store.getJob(fake.job.id)?.model_json ?? null) + assert.deepEqual(run?.attempts.map((entry) => [entry.modelId, entry.status, entry.retired]), [['pvm_a', 404, true]]) + assert.equal(run?.model?.label, 'Backup Model', 'the job still names the model that actually answered') + + // The row is the record the pages read. + const rows = fake.store.listProviderModels(fake.providerA.id) + const rowA = rows.find((row) => row.id === fake.modelA.id) + assert.notEqual(rowA?.retired_at, null, 'the retired model records the date it was retired') + assert.notEqual(rowA?.retired_at, undefined, 'the retired model records the date it was retired') + const retiredAt = String(rowA?.retired_at) + const date = retiredAt.slice(0, 10) + assert.deepEqual(fake.store.listProviderModels(fake.providerB.id).map((row) => row.retired_at), [null], 'a model that answered is not retired') + + // Settings: the model is still listed, now labelled with the date. + const settings = await page(fake.built, fake.user, '/settings') + assert.match(settings, new RegExp(`data-retired="${date}"`), 'Settings carries the retired date for the model') + assert.equal(settings.split('data-retired=').length - 1, 1, 'only the retired model carries the marker') + + // Repository page: the retired default is marked. + const repoPage = await page(fake.built, fake.user, `/repos/${fake.repo.id}`) + assert.match(repoPage, /data-retired-model="pvm_a"/, 'the repository default marks the retired model') + assert.doesNotMatch(repoPage, /data-retired-model="pvm_b"/, 'a live model is not marked') + + // Job page: the model line names the reason. + const jobPage = await page(fake.built, fake.user, `/repos/${fake.repo.id}/jobs/${fake.job.id}`) + assert.match(jobPage, /data-retired-attempt="pvm_a"/, 'the job model line names the retired attempt') + + // One retirement, one date: a second call that gets the same 404 must not move it. + const second = await fake.call() + assert.equal(second.statusCode, 200) + const after = fake.store.listProviderModels(fake.providerA.id).find((row) => row.id === fake.modelA.id) + assert.equal(after?.retired_at, retiredAt, 'a second 404 does not move the retirement date') + } finally { + await fake.close() + } +}) + +test('a rate limit or a provider error never retires a model, and the fallback still works', async () => { + let fallbackStatus = 200 + const fake = await fixture( + chatHit([], () => ({ status: 429, payload: { error: { message: 'Rate limit reached for requests' } } })), + chatHit([], () => (fallbackStatus === 200 ? OK : { status: 500, payload: { error: { message: 'upstream exploded' } } })), + ) + try { + const response = await fake.call() + assert.equal(response.statusCode, 200, 'a rate-limited primary still finishes on the fallback') + + let run = parseModelRun(fake.store.getJob(fake.job.id)?.model_json ?? null) + assert.deepEqual(run?.attempts.map((entry) => [entry.status, entry.retryable, entry.retired]), [[429, true, undefined]]) + assert.deepEqual(fake.store.listProviderModels(fake.providerA.id).map((row) => row.retired_at), [null], 'a 429 is not a retirement') + assert.deepEqual(fake.store.listProviderModels(fake.providerB.id).map((row) => row.retired_at), [null], 'an answering model is not retired') + + fallbackStatus = 500 + const failed = await fake.call() + assert.equal(failed.statusCode, 500, 'a 500 on the last model fails the call as before') + run = parseModelRun(fake.store.getJob(fake.job.id)?.model_json ?? null) + assert.deepEqual(run?.attempts.map((entry) => [entry.status, entry.retired]), [[429, undefined], [500, undefined]], 'neither transient status records a retirement') + assert.deepEqual(fake.store.listProviderModels(fake.providerA.id).map((row) => row.retired_at), [null], 'a 500 is not a retirement either') + + const settings = await page(fake.built, fake.user, '/settings') + assert.doesNotMatch(settings, /data-retired=/, 'no model is labelled retired') + } finally { + await fake.close() + } +}) + +test('an apply with a retired model selected is refused by name, and a live selection passes it', async () => { + const fake = await fixture( + chatHit([], () => RETIRED), + chatHit([], () => OK), + ) + try { + assert.equal((await fake.call()).statusCode, 200) + const { cookie, csrf } = sessionFor(fake.built, fake.user) + const headers = { cookie, 'content-type': 'application/x-www-form-urlencoded' } + const rejected = await fake.built.app.inject({ method: 'POST', url: `/repos/${fake.repo.id}/models/apply`, headers, payload: `csrf=${encodeURIComponent(csrf)}` }) + assert.equal(rejected.statusCode, 302, 'the route answers with its existing redirect') + // #472 slice 2: the message rides a one-shot cookie, not the Location query. + const refused = flashOf(rejected)?.error?.text ?? '' + assert.match(refused, /GPT 4o/, 'the refusal names the retired model') + assert.match(refused, /(下架|retired)/, 'the refusal says why') + assert.equal(fake.store.listAudit(100).filter((entry) => entry.action === 'repo.models.applied').length, 0, 'nothing was applied or restarted') + + // The maintainer picks the live model; the same route is no longer refused for retirement. + fake.store.setRepoModelSelection(fake.repo.id, [fake.modelB.id], fake.modelB.id) + const allowed = await fake.built.app.inject({ method: 'POST', url: `/repos/${fake.repo.id}/models/apply`, headers, payload: `csrf=${encodeURIComponent(csrf)}` }) + assert.equal(allowed.statusCode, 302) + assert.doesNotMatch(flashOf(allowed)?.error?.text ?? '', /(下架|retired)/, 'a live selection is not refused for retirement') + } finally { + await fake.close() + } +}) diff --git a/test/notificationOutbox.test.ts b/test/notificationOutbox.test.ts index 7ef51db2..856ef997 100644 --- a/test/notificationOutbox.test.ts +++ b/test/notificationOutbox.test.ts @@ -42,11 +42,11 @@ function row(id: string): NotificationOutboxRow { return { id, payload_json: JSON.stringify({ event: 'ops.alert', text: 'disk almost full' }) } as NotificationOutboxRow } -test('the outbox table and its state machine exist at user_version 44', () => { +test('the outbox table and its state machine exist at user_version 45', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 44, 'the notification outbox moves the refusal boundary from 43 to 44') + assert.equal(Number(version?.user_version), 45, 'the retired provider model moves the refusal boundary from 44 to 45') const columns = store.all<{ name: string }>('PRAGMA table_info(notification_outbox)').map((column) => column.name) for (const column of ['transport', 'destination', 'payload_json', 'idempotency_key', 'state', 'attempts', 'next_attempt_at', 'lease_token', 'lease_expires_at', 'last_error', 'sent_at']) { assert.equal(columns.includes(column), true, `notification_outbox must have ${column}`) diff --git a/test/scheduleLeases.test.ts b/test/scheduleLeases.test.ts index 3eae9456..8180d588 100644 --- a/test/scheduleLeases.test.ts +++ b/test/scheduleLeases.test.ts @@ -45,11 +45,11 @@ function claim(h: { store: Store; repoId: string; scheduleId: string }, token: s }) } -test('the occurrence row carries attempts and a lease, at user_version 44', () => { +test('the occurrence row carries attempts and a lease, at user_version 45', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 44, 'the notification outbox moves the refusal boundary from 43 to 44') + assert.equal(Number(version?.user_version), 45, 'the retired provider model moves the refusal boundary from 44 to 45') const columns = store.all<{ name: string }>('PRAGMA table_info(schedule_occurrences)').map((column) => column.name) for (const column of ['attempts', 'lease_token', 'lease_expires_at']) { diff --git a/test/scheduleOccurrences.test.ts b/test/scheduleOccurrences.test.ts index efd44062..35d6cb8a 100644 --- a/test/scheduleOccurrences.test.ts +++ b/test/scheduleOccurrences.test.ts @@ -56,11 +56,11 @@ function audits(store: Store, action: string): string[] { // -- the table ------------------------------------------------------------- -test('the occurrence table is part of the schema at user_version 44', () => { +test('the occurrence table is part of the schema at user_version 45', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 44, 'the notification outbox moves the refusal boundary from 43 to 44') + assert.equal(Number(version?.user_version), 45, 'the retired provider model moves the refusal boundary from 44 to 45') const table = store.get<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schedule_occurrences'") assert.equal(table?.name, 'schedule_occurrences') diff --git a/test/scheduleSchema.test.ts b/test/scheduleSchema.test.ts index de3729a7..ddef9164 100644 --- a/test/scheduleSchema.test.ts +++ b/test/scheduleSchema.test.ts @@ -7,11 +7,11 @@ import { makeUser } from './helpers.js' // the version boundary, the columns the planner reads, and the one-row-per- // repository rule the tick relies on. -test('the schedule table is part of the schema at user_version 44', () => { +test('the schedule table is part of the schema at user_version 45', () => { const store = new Store(':memory:') try { const version = store.get<{ user_version: number }>('PRAGMA user_version') - assert.equal(Number(version?.user_version), 44, 'each schema change moves the refusal boundary: 42 triage, 43 Lead delivery, 44 notification outbox') + assert.equal(Number(version?.user_version), 45, 'each schema change moves the refusal boundary: 42 triage, 43 Lead delivery, 44 notification outbox, 45 retired provider model') const table = store.get<{ name: string }>("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'schedules'") assert.equal(table?.name, 'schedules')