From 8775b018cfb71b170b883f3b06a1fa168bc527f8 Mon Sep 17 00:00:00 2001 From: Harlan GitHub Agent Date: Thu, 12 Nov 2026 17:47:15 +1100 Subject: [PATCH] perf(artifact): drain several Resolution prune batches per sweep One batch of 500 every 15 minutes needs about 2.5 days to clear the 110k Resolutions left from the November capacity incident, and the database stays near its 10 GB cap meanwhile. Prune up to eight batches per sweep within a 20-second budget, stopping at the first short batch, so the backlog clears in hours and steady state still runs one query. Co-Authored-By: Claude Opus 5.5 --- .../server/tasks/sweep-skill-runs.ts | 4 +-- .../server/utils/resolution-retention.ts | 28 +++++++++++++++++++ test/unit/resolution-retention.test.ts | 19 ++++++++++++- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts b/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts index d95f868b..1f609def 100644 --- a/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts +++ b/layers/artifact-delivery/server/tasks/sweep-skill-runs.ts @@ -7,7 +7,7 @@ import { reportJobRun } from '~~/server/utils/sync-job-reporter' import { observedSchedulePolicy } from '#shared/schedule-policy' import { enqueueArtifactBuild } from '../utils/queue' import { fetchAdmittedSkillIdentity, requestResolution } from '../utils/request-resolution' -import { pruneSettledResolutions } from '../utils/resolution-retention' +import { drainSettledResolutions } from '../utils/resolution-retention' import { runSkillRunSweep } from '../utils/run-sweep' const CRON = '7,22,37,52 * * * *' @@ -47,7 +47,7 @@ export default defineScheduledTask({ // Prune before the observed run records its start: a full database refuses // that insert, and only a delete can free the space it needs. - const pruned = await pruneSettledResolutions(db, { now: Math.floor(Date.now() / 1000) }) + const pruned = await drainSettledResolutions(db, { now: Math.floor(Date.now() / 1000), clockMs: Date.now }) emitOperationalEvent(createWideEvent({ 'operation': 'prune-resolutions', 'outcome': 'completed', 'processed.count': pruned }), 'info') return await runObservedScheduledTask({ diff --git a/layers/artifact-delivery/server/utils/resolution-retention.ts b/layers/artifact-delivery/server/utils/resolution-retention.ts index e4b2c550..34ce18f1 100644 --- a/layers/artifact-delivery/server/utils/resolution-retention.ts +++ b/layers/artifact-delivery/server/utils/resolution-retention.ts @@ -98,3 +98,31 @@ export async function deleteUnneededResolution(db: D1Database, id: string): Prom ).bind(id).run() return (result.meta.changes ?? 0) > 0 } + +/** Batches one drain may run. Eight batches clear a 100k backlog in about 6 hours of sweeps. */ +export const RESOLUTION_DRAIN_BATCHES = 8 + +/** Time one drain may spend before it leaves the rest to the next run. */ +export const RESOLUTION_DRAIN_BUDGET_MS = 20_000 + +/** + * Prune batches until one comes back short, the batch cap is hit, or the time + * budget runs out. Steady state stops after one short batch. Returns how many + * Resolutions it deleted. + */ +export async function drainSettledResolutions( + db: D1Database, + input: { now: number, clockMs: () => number, limit?: number, maxBatches?: number, budgetMs?: number }, +): Promise { + const limit = input.limit ?? RESOLUTION_PRUNE_BATCH + const maxBatches = input.maxBatches ?? RESOLUTION_DRAIN_BATCHES + const deadline = input.clockMs() + (input.budgetMs ?? RESOLUTION_DRAIN_BUDGET_MS) + let deleted = 0 + for (let batch = 0; batch < maxBatches && input.clockMs() < deadline; batch++) { + const count = await pruneSettledResolutions(db, { now: input.now, limit }) + deleted += count + if (count < limit) + break + } + return deleted +} diff --git a/test/unit/resolution-retention.test.ts b/test/unit/resolution-retention.test.ts index aba5abae..bff1ca71 100644 --- a/test/unit/resolution-retention.test.ts +++ b/test/unit/resolution-retention.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import type { SqliteD1 } from './helpers/d1-sqlite' import { afterEach, describe, expect, it } from 'vitest' -import { deleteUnneededResolution, pruneSettledResolutions, RESOLUTION_RETENTION_SECONDS } from '../../layers/artifact-delivery/server/utils/resolution-retention' +import { deleteUnneededResolution, drainSettledResolutions, pruneSettledResolutions, RESOLUTION_RETENTION_SECONDS } from '../../layers/artifact-delivery/server/utils/resolution-retention' import { allMigrations, createSqliteD1 } from './helpers/d1-sqlite' const NOW = 1_795_000_000 @@ -134,4 +134,21 @@ describe('pruneSettledResolutions', () => { expect(await pruneSettledResolutions(sqlite.db, { now: NOW, limit: 2 })).toBe(1) expect(remaining()).toEqual([]) }) + + it('drains batches until one comes back short', async () => { + seed(['f1', 'f2', 'f3', 'f4', 'f5'].map(id => ({ id, at: OLD, state: 'failed' as const }))) + + expect(await drainSettledResolutions(sqlite.db, { now: NOW, clockMs: () => 0, limit: 2 })).toBe(5) + expect(remaining()).toEqual([]) + }) + + it('stops at the batch cap and when the time budget runs out', async () => { + seed(['f1', 'f2', 'f3', 'f4', 'f5'].map(id => ({ id, at: OLD, state: 'failed' as const }))) + + expect(await drainSettledResolutions(sqlite.db, { now: NOW, clockMs: () => 0, limit: 2, maxBatches: 1 })).toBe(2) + let clock = 0 + const slow = () => (clock += 15_000) + expect(await drainSettledResolutions(sqlite.db, { now: NOW, clockMs: slow, limit: 1, budgetMs: 20_000 })).toBe(1) + expect(remaining()).toHaveLength(2) + }) })