Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 33 additions & 9 deletions layers/artifact-delivery/server/utils/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ import {
LEADING_BUILD_FRESH_SECONDS,
parseCheckResults,
publishArtifactRecord,
shareReusedAttestation,
transitionResolution,
} from './state'
import { readDeterministicUstar } from './ustar'
Expand Down Expand Up @@ -311,6 +312,21 @@ export async function processArtifactBuild(
if (loaded._tag === 'rejected')
return await failWithRejection(dependencies, row, loaded.rejection)
carried = loaded
if (loaded._tag === 'reused' && !reusedContentChanged(content, loaded.build)) {
// A reused build serves the attestation of the build it reused and
// stores no copy. If that attestation is gone, it signs its own.
const shared = await shareReusedAttestation(dependencies.db, row, {
attestationResolutionId: loaded.build.resolutionId,
r2Key: artifactR2Key(content.contentSha256),
now,
})
if (shared._tag === 'superseded')
return { _tag: 'superseded', resolutionId }
if (shared._tag === 'advanced') {
row = shared.row
continue
}
}
const statement = loaded._tag === 'reused'
? stageReusedStatement(row, content, loaded.build)
: await stageLoadedStatement(dependencies, row, content, loaded)
Expand Down Expand Up @@ -343,11 +359,12 @@ export async function processArtifactBuild(
}

if (row.state === 'publishing') {
const attestation = artifactAttestationSchema.parse(JSON.parse(row.attestation_json ?? 'null'))
const published = await publishArtifactRecord(dependencies.db, {
row,
checks: parseCheckResults(row.check_results_json),
attestation,
attestation: row.attestation_resolution_id
? { _tag: 'shared' }
: { _tag: 'signed', attestation: artifactAttestationSchema.parse(JSON.parse(row.attestation_json ?? 'null')) },
now,
})
return published._tag === 'superseded'
Expand Down Expand Up @@ -444,7 +461,9 @@ async function stageLoadedStatement(
}

/**
* Stage a new statement over bytes that R2 already holds.
* Stage a new statement over bytes that R2 already holds. A reused build
* comes here only when the build it reused no longer holds its attestation;
* otherwise it serves that one (see `shareReusedAttestation`).
*
* The statement names this Resolution's source and creation time, with the
* files and check results of the reused build. The signer then signs it
Expand All @@ -457,13 +476,8 @@ function stageReusedStatement(
content: { artifactId: string, contentSha256: string, contentBytes: number },
build: ReusableBuild,
): StagedStatement {
if (
build.artifactId !== content.artifactId
|| build.contentSha256 !== content.contentSha256
|| build.contentBytes !== content.contentBytes
) {
if (reusedContentChanged(content, build))
return { _tag: 'content-changed' }
}
const statement = createAttestationStatement({
artifactId: content.artifactId,
createdAt: new Date(row.created_at * 1000).toISOString(),
Expand All @@ -483,6 +497,16 @@ function stageReusedStatement(
}
}

/** Whether a reused build holds other bytes than this Resolution recorded. */
function reusedContentChanged(
content: { artifactId: string, contentSha256: string, contentBytes: number },
build: ReusableBuild,
): boolean {
return build.artifactId !== content.artifactId
|| build.contentSha256 !== content.contentSha256
|| build.contentBytes !== content.contentBytes
}

/**
* A pinned public request, before any GitHub read: reuse a ready build of its
* commit, check stored bytes again, or wait for an earlier build of it.
Expand Down
8 changes: 5 additions & 3 deletions layers/artifact-delivery/server/utils/grant.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ export interface PublicGrantDependencies {

/**
* Every grant reads current D1 check results before it reveals cached bytes.
* A named Resolution that reused a build is granted under its own state and
* checks, with the attestation of the build it names.
* The public R2 URL is immutable. A cached object cannot create a fresh grant.
*/
export async function createPublicArtifactGrant(
Expand All @@ -60,13 +62,13 @@ export async function createPublicArtifactGrant(
const rowStatement = dependencies.db.prepare(
`SELECT
a.id, a.content_sha256, a.content_bytes, a.r2_key, a.delivery_status,
aa.resolution_id, aa.attestation_json,
r.id AS resolution_id, aa.attestation_json,
r.state AS resolution_state
FROM artifacts a
JOIN artifact_attestations aa ON aa.artifact_id = a.id
JOIN artifact_resolutions r ON r.id = aa.resolution_id
JOIN artifact_resolutions r ON r.id = COALESCE(?2, aa.resolution_id)
WHERE a.id = ?1
AND (?2 IS NULL OR aa.resolution_id = ?2)
AND aa.resolution_id = COALESCE(r.attestation_resolution_id, r.id)
ORDER BY aa.created_at DESC, aa.resolution_id DESC
LIMIT 1`,
).bind(artifactId, resolutionId ?? null)
Expand Down
12 changes: 10 additions & 2 deletions layers/artifact-delivery/server/utils/resolution-retention.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,12 @@ export const RESOLUTION_PRUNE_BATCH = 500
* - a run check still reads it;
* - a download grant names it;
* - it is ready and holds the newest attestation of its Artifact, which build
* reuse copies and the Artifact routes serve.
* reuse reads and the Artifact routes serve;
* - a reused build names it, because that build serves its attestation.
*
* Deleting one cascades to its attestation, check results, and requester rows.
* A reused build holds no attestation of its own, so it goes once the window
* passes. Deleting one cascades to its attestation, check results, and
* requester rows. Migration 0161 indexes the reused-build lookup.
* The batch prune filters by `idx_artifact_resolutions_state`, so its scan reads
* only settled rows older than the window.
*/
Expand All @@ -40,8 +43,13 @@ const UNNEEDED = `
UNION
SELECT resolution_id FROM artifact_download_grants
)
AND NOT EXISTS (
SELECT 1 FROM artifact_resolutions reused
WHERE reused.attestation_resolution_id = r.id
)
AND (
r.state <> 'ready'
OR r.attestation_resolution_id IS NOT NULL
OR EXISTS (
SELECT 1
FROM artifact_attestations own
Expand Down
106 changes: 91 additions & 15 deletions layers/artifact-delivery/server/utils/state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ const resolutionRowSchema = z.object({
check_results_json: z.string().nullable(),
attestation_statement_json: z.string().nullable(),
attestation_json: z.string().nullable(),
/**
* A reused build stores no attestation. It names the Resolution that signed
* the one it serves. See migration 0161.
*/
attestation_resolution_id: z.string().nullable().optional(),
/** The attestation of `attestation_resolution_id`, from {@link RESOLUTION_SELECT}. */
shared_attestation_json: z.string().nullable().optional(),
visibility: z.enum(['public', 'private']),
account_id: z.number().int().positive().nullable(),
github_installation_id: z.number().int().positive().safe().nullable(),
Expand Down Expand Up @@ -226,16 +233,24 @@ export async function createResolution(
: { _tag: 'idempotency-conflict' }
}

/**
* A Resolution row with the attestation it serves. A reused build reads the
* attestation of the Resolution it names.
*/
const RESOLUTION_SELECT = `SELECT r.*, shared.attestation_json AS shared_attestation_json
FROM artifact_resolutions r
LEFT JOIN artifact_resolutions shared ON shared.id = r.attestation_resolution_id`

async function findResolutionByRequestKey(db: D1Database, keyHash: string): Promise<ResolutionRow | null> {
const row = await db.prepare(
'SELECT * FROM artifact_resolutions WHERE request_key_hash = ?1 LIMIT 1',
`${RESOLUTION_SELECT} WHERE r.request_key_hash = ?1 LIMIT 1`,
).bind(keyHash).first<Record<string, unknown>>()
return row ? resolutionRowSchema.parse(row) : null
}

export async function getResolution(db: D1Database, resolutionId: string): Promise<ResolutionRow | null> {
const row = await db.prepare(
'SELECT * FROM artifact_resolutions WHERE id = ?1 LIMIT 1',
`${RESOLUTION_SELECT} WHERE r.id = ?1 LIMIT 1`,
).bind(resolutionId).first<Record<string, unknown>>()
return row ? resolutionRowSchema.parse(row) : null
}
Expand Down Expand Up @@ -313,6 +328,7 @@ export type ReadyBuildLookup

/** A ready public build as D1 stores it. Nothing here verifies its attestation. */
export interface ReadyPublicBuild {
/** The Resolution that signed {@link attestationJson}. A reused build names it. */
resolutionId: string
source: ResolvedSource
artifactId: string
Expand All @@ -334,7 +350,8 @@ const readyPublicBuildRowSchema = z.object({
content_sha256: z.string(),
content_bytes: z.number().int().positive(),
r2_key: z.string(),
attestation_json: z.string(),
attestation_resolution_id: z.string().uuid(),
attestation_json: z.string().nullable(),
})

/**
Expand All @@ -355,8 +372,12 @@ export async function findReadyPublicBuild(
if (!value)
return null
const row = readyPublicBuildRowSchema.parse(value)
// A reused build whose signed Resolution is gone serves nothing. The prune
// keeps every Resolution a reused build names, so this is a fault.
if (row.attestation_json === null)
return null
return {
resolutionId: row.id,
resolutionId: row.attestation_resolution_id,
source: resolvedSourceSchema.parse({
provider: 'github',
repositoryId: row.repository_id,
Expand Down Expand Up @@ -386,9 +407,12 @@ export async function findReadyPublicBuild(
// instead of every ready row of the commit.
const READY_PUBLIC_BUILD_SELECT = `SELECT r.id, r.repository_id, r.resolved_owner, r.resolved_repository,
r.commit_sha, r.tree_sha, r.skill_path, r.artifact_id, r.content_sha256,
r.content_bytes, r.r2_key, r.attestation_json
r.content_bytes, r.r2_key,
COALESCE(r.attestation_resolution_id, r.id) AS attestation_resolution_id,
COALESCE(r.attestation_json, shared.attestation_json) AS attestation_json
FROM artifact_resolutions r
JOIN artifacts a ON a.id = r.artifact_id
LEFT JOIN artifact_resolutions shared ON shared.id = r.attestation_resolution_id
WHERE r.state = 'ready' AND r.visibility = 'public'
AND a.delivery_status = 'available'`
const NEWEST_READY_BUILD = 'ORDER BY r.created_at DESC, r.id DESC LIMIT 1'
Expand Down Expand Up @@ -520,18 +544,59 @@ export function presentResolution(row: ResolutionRow): ResolutionResponse {
artifact: {
artifactId: row.artifact_id,
visibility: row.visibility,
attestation: parseAttestation(row.attestation_json),
attestation: parseAttestation(row.attestation_json ?? row.shared_attestation_json ?? null),
},
}
}

/**
* The attestation a published Resolution serves: one it signed, or the one
* of the build it reused, which it does not store again.
*/
export type PublishedAttestation
= { _tag: 'signed', attestation: ArtifactAttestation }
| { _tag: 'shared' }

export interface PublishArtifactInput {
row: ResolutionRow
checks: CheckResult[]
attestation: ArtifactAttestation
attestation: PublishedAttestation
now: number
}

/**
* Move a signing Resolution that reuses a ready public build to `publishing`,
* naming the Resolution that signed the attestation it serves. It stores no
* attestation and asks the signer for none.
*
* The write applies only while the named Resolution still holds its
* attestation, so a prune between the reuse decision and this write cannot
* leave the Resolution naming nothing. Once written, the prune keeps the named
* Resolution. `attestation-missing` means the build must sign its own.
*/
export async function shareReusedAttestation(
db: D1Database,
row: ResolutionRow,
input: { attestationResolutionId: string, r2Key: string, now: number },
): Promise<{ _tag: 'advanced', row: ResolutionRow } | { _tag: 'superseded' | 'attestation-missing' }> {
if (row.state !== 'signing')
throw new Error(`Invalid Artifact build transition: ${row.state} to publishing`)
const advanced = await db.prepare(
`UPDATE artifact_resolutions
SET state = 'publishing', state_version = state_version + 1, updated_at = ?1,
r2_key = ?2, attestation_resolution_id = ?3
WHERE id = ?4 AND state = 'signing' AND state_version = ?5
AND EXISTS (SELECT 1 FROM artifact_attestations WHERE resolution_id = ?3)
RETURNING *`,
).bind(input.now, input.r2Key, input.attestationResolutionId, row.id, row.state_version).first<Record<string, unknown>>()
if (advanced)
return { _tag: 'advanced', row: resolutionRowSchema.parse(advanced) }
const current = await getResolution(db, row.id)
return current?.state === 'signing' && current.state_version === row.state_version
? { _tag: 'attestation-missing' }
: { _tag: 'superseded' }
}

export async function publishArtifactRecord(
db: D1Database,
input: PublishArtifactInput,
Expand All @@ -553,8 +618,10 @@ export async function publishArtifactRecord(
}
}

if (attestation._tag === 'shared' && (row.visibility !== 'public' || !row.attestation_resolution_id))
throw new Error('Only a reused public build serves a shared attestation')
const artifactStatements: D1PreparedStatement[] = row.visibility === 'private'
? privateArtifactPublishStatements(db, row, attestation, now)
? privateArtifactPublishStatements(db, row, signedAttestation(attestation), now)
: publicArtifactPublishStatements(db, row, attestation, now)
const statements: D1PreparedStatement[] = [
...artifactStatements,
Expand Down Expand Up @@ -586,23 +653,32 @@ export async function publishArtifactRecord(
return { _tag: 'published', row: resolutionRowSchema.parse(published) }
}

function signedAttestation(attestation: PublishedAttestation): ArtifactAttestation {
if (attestation._tag !== 'signed')
throw new Error('A private Artifact signs its own attestation')
return attestation.attestation
}

function publicArtifactPublishStatements(
db: D1Database,
row: ResolutionRow,
attestation: ArtifactAttestation,
attestation: PublishedAttestation,
now: number,
): D1PreparedStatement[] {
const artifact = db.prepare(
`INSERT OR IGNORE INTO artifacts (
id, content_sha256, content_bytes, format, r2_key, delivery_status, created_at, updated_at
) VALUES (?1, ?2, ?3, 'skilld-tar-v1', ?4, 'available', ?5, ?5)`,
).bind(row.artifact_id!, row.content_sha256!, row.content_bytes!, row.r2_key!, now)
if (attestation._tag === 'shared')
return [artifact]
return [
db.prepare(
`INSERT OR IGNORE INTO artifacts (
id, content_sha256, content_bytes, format, r2_key, delivery_status, created_at, updated_at
) VALUES (?1, ?2, ?3, 'skilld-tar-v1', ?4, 'available', ?5, ?5)`,
).bind(row.artifact_id!, row.content_sha256!, row.content_bytes!, row.r2_key!, now),
artifact,
db.prepare(
`INSERT OR REPLACE INTO artifact_attestations (
resolution_id, artifact_id, attestation_json, created_at
) VALUES (?1, ?2, ?3, ?4)`,
).bind(row.id, row.artifact_id!, JSON.stringify(attestation), now),
).bind(row.id, row.artifact_id!, JSON.stringify(attestation.attestation), now),
]
}

Expand Down
17 changes: 17 additions & 0 deletions migrations/0161_artifact_resolution_shared_attestation.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
-- The Resolution whose attestation a reused build serves.
--
-- A build that reuses a ready public build used to sign a new statement and
-- store another copy of the same attestation: about 33 KB on the row and
-- 23 KB in `artifact_attestations`. In a 12-hour sample on 2026-11-12, 1,967
-- ready Resolutions covered 142 Artifacts. A reused build now stores no
-- attestation. It names the Resolution that holds one, and readers serve that.
--
-- It always names a Resolution that signed its own attestation, never another
-- reused build. NULL for every build that signed its own.
ALTER TABLE artifact_resolutions ADD COLUMN attestation_resolution_id TEXT;

-- The prune keeps a Resolution while a reused build names it. Only reused
-- builds enter this index.
CREATE INDEX IF NOT EXISTS idx_artifact_resolutions_attestation_resolution
ON artifact_resolutions(attestation_resolution_id)
WHERE attestation_resolution_id IS NOT NULL;
Loading
Loading