diff --git a/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts b/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts index d95f868b..1f609def 100644 --- a/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts +++ b/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts @@ -7,7 +7,7 @@ import { reportJobRun } from '~~/server/utils/sync-job-reporter' import { observedSchedulePolicy } from '#shared/schedule-policy' import { enqueueArtifactBuild } from '../utils/queue' import { fetchAdmittedSkillIdentity, requestResolution } from '../utils/request-resolution' -import { pruneSettledResolutions } from '../utils/resolution-retention' +import { drainSettledResolutions } from '../utils/resolution-retention' import { runSkillRunSweep } from '../utils/run-sweep' const CRON = '7,22,37,52 * * * *' @@ -47,7 +47,7 @@ export default defineScheduledTask({ // Prune before the observed run records its start: a full database refuses // that insert, and only a delete can free the space it needs. - const pruned = await pruneSettledResolutions(db, { now: Math.floor(Date.now() / 1000) }) + const pruned = await drainSettledResolutions(db, { now: Math.floor(Date.now() / 1000), clockMs: Date.now }) emitOperationalEvent(createWideEvent({ 'operation': 'prune-resolutions', 'outcome': 'completed', 'processed.count': pruned }), 'info') return await runObservedScheduledTask({ diff --git a/layers/artifact-delivery/server/utils/resolution-retention.ts b/layers/artifact-delivery/server/utils/resolution-retention.ts index e4b2c550..34ce18f1 100644 --- a/layers/artifact-delivery/server/utils/resolution-retention.ts +++ b/layers/artifact-delivery/server/utils/resolution-retention.ts @@ -98,3 +98,31 @@ export async function deleteUnneededResolution(db: D1Database, id: string): Prom ).bind(id).run() return (result.meta.changes ?? 0) > 0 } + +/** Batches one drain may run. Eight batches clear a 100k backlog in about 6 hours of sweeps. */ +export const RESOLUTION_DRAIN_BATCHES = 8 + +/** Time one drain may spend before it leaves the rest to the next run. */ +export const RESOLUTION_DRAIN_BUDGET_MS = 20_000 + +/** + * Prune batches until one comes back short, the batch cap is hit, or the time + * budget runs out. Steady state stops after one short batch. Returns how many + * Resolutions it deleted. + */ +export async function drainSettledResolutions( + db: D1Database, + input: { now: number, clockMs: () => number, limit?: number, maxBatches?: number, budgetMs?: number }, +): Promise { + const limit = input.limit ?? RESOLUTION_PRUNE_BATCH + const maxBatches = input.maxBatches ?? RESOLUTION_DRAIN_BATCHES + const deadline = input.clockMs() + (input.budgetMs ?? RESOLUTION_DRAIN_BUDGET_MS) + let deleted = 0 + for (let batch = 0; batch < maxBatches && input.clockMs() < deadline; batch++) { + const count = await pruneSettledResolutions(db, { now: input.now, limit }) + deleted += count + if (count < limit) + break + } + return deleted +} diff --git a/test/unit/resolution-retention.test.ts b/test/unit/resolution-retention.test.ts index aba5abae..bff1ca71 100644 --- a/test/unit/resolution-retention.test.ts +++ b/test/unit/resolution-retention.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import type { SqliteD1 } from './helpers/d1-sqlite' import { afterEach, describe, expect, it } from 'vitest' -import { deleteUnneededResolution, pruneSettledResolutions, RESOLUTION_RETENTION_SECONDS } from '../../layers/artifact-delivery/server/utils/resolution-retention' +import { deleteUnneededResolution, drainSettledResolutions, pruneSettledResolutions, RESOLUTION_RETENTION_SECONDS } from '../../layers/artifact-delivery/server/utils/resolution-retention' import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' const NOW = 1_795_000_000 @@ -134,4 +134,21 @@ describe('pruneSettledResolutions', () => { expect(await pruneSettledResolutions(sqlite.db, { now: NOW, limit: 2 })).toBe(1) expect(remaining()).toEqual([]) }) + + it('drains batches until one comes back short', async () => { + seed(['f1', 'f2', 'f3', 'f4', 'f5'].map(id => ({ id, at: OLD, state: 'failed' as const }))) + + expect(await drainSettledResolutions(sqlite.db, { now: NOW, clockMs: () => 0, limit: 2 })).toBe(5) + expect(remaining()).toEqual([]) + }) + + it('stops at the batch cap and when the time budget runs out', async () => { + seed(['f1', 'f2', 'f3', 'f4', 'f5'].map(id => ({ id, at: OLD, state: 'failed' as const }))) + + expect(await drainSettledResolutions(sqlite.db, { now: NOW, clockMs: () => 0, limit: 2, maxBatches: 1 })).toBe(2) + let clock = 0 + const slow = () => (clock += 15_000) + expect(await drainSettledResolutions(sqlite.db, { now: NOW, clockMs: slow, limit: 1, budgetMs: 20_000 })).toBe(1) + expect(remaining()).toHaveLength(2) + }) })