From 0f7ea6829142d931f7633c45057249f9c8213653 Mon Sep 17 00:00:00 2001 From: Day Dreamer <87640074+HugoXK@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:22:20 +0800 Subject: [PATCH] fix: stop starting work on a host whose clock is measurably wrong (#758) --- changelog.d/758-clock-skew.fixed.md | 1 + docs/api.md | 36 ++++- docs/architecture.md | 21 ++- docs/deploy.md | 22 +++ src/app.ts | 19 ++- src/jobs/service.ts | 17 ++ src/lib/failure.ts | 6 + src/repos/app.ts | 90 ++++++++--- test/clockSkew.test.ts | 22 ++- test/clockSkewFail.test.ts | 230 ++++++++++++++++++++++++++++ 10 files changed, 418 insertions(+), 46 deletions(-) create mode 100644 changelog.d/758-clock-skew.fixed.md create mode 100644 test/clockSkewFail.test.ts diff --git a/changelog.d/758-clock-skew.fixed.md b/changelog.d/758-clock-skew.fixed.md new file mode 100644 index 00000000..bcd5b47a --- /dev/null +++ b/changelog.d/758-clock-skew.fixed.md @@ -0,0 +1 @@ +The measured clock offset is now used, not only reported. Above the same 120 seconds the alert rule fires on, `/readyz` answers 503 with `clock.state: "fail"`, `Jobs.enqueue` refuses with `CLOCK_SKEW` before any job row is written (counted as `fledge_admission_rejected_total{reason="clock_skew"}`), and the App JWT refuses with `CLOCK_SKEW_EXCEEDED` instead of signing from a stale correction. A reading is trusted for fifteen minutes, after which the clock is `unknown` and signing fails with `CLOCK_SKEW_UNVERIFIED`, so a GitHub outage can no longer keep a wrong correction alive. Stored timestamps are unchanged and a wrong host clock still needs the host fix. diff --git a/docs/api.md b/docs/api.md index a7e1b829..7d7f4ab4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -732,7 +732,7 @@ Public, and answerable without a token, because a readiness probe that needs a credential fails when the credential does. The body is: ```json -{ "ok": true, "db": true, "instances": { "ready": 2, "failed": 0, "total": 2 }, "clock": { "skewSeconds": 1680, "source": "github" } } +{ "ok": false, "db": true, "instances": { "ready": 2, "failed": 0, "total": 2 }, "clock": { "skewSeconds": 1680, "measuredAt": "2026-11-15T21:14:02.000Z", "thresholdSeconds": 120, "state": "fail", "source": "github" } } ``` `instances` counts the per-repository instances the database knows about. @@ -750,14 +750,36 @@ API, and an outage cannot make readiness flap. An operator watches an absent `fledge_clock_skew_seconds` series as "not measured", not as "aligned" (see [`docs/deploy.md`](deploy.md)). +`clock.state` is the decision the measurement supports, and `ok` in the body +follows it: + +| state | meaning | response | +|---|---|---| +| `not-configured` | this deployment has no GitHub App, so nothing is measured | `200` | +| `ok` | measured at most 15 minutes ago and within `thresholdSeconds` | `200` | +| `unknown` | never measured, or the last reading is older than 15 minutes | `200` | +| `fail` | the last measurement is outside `thresholdSeconds` | **`503`, `ok:false`** | + +`clock.thresholdSeconds` is `CLOCK_SKEW_FAIL_SECONDS` (`src/repos/app.ts`), the +same `120` the `FledgeClockSkewHigh` alert fires on, and `clock.measuredAt` is +when the reported reading was taken. `unknown` and `not-configured` stay `200` +on purpose: a GitHub outage, or a deployment without an App, is not a clock +verdict. In `fail` the platform also refuses work — `Jobs.enqueue` answers +`CLOCK_SKEW` before any job row is written, counted as +`fledge_admission_rejected_total{reason="clock_skew"}` — and the App JWT refuses +to sign (`CLOCK_SKEW_EXCEEDED`), where `unknown` refuses with +`CLOCK_SKEW_UNVERIFIED`. + The measurement itself lives on the GitHub App client (`src/repos/app.ts`): `measureSkew()` reads the `Date` header of one -unauthenticated `GET /rate_limit`, records it, and leaves `null` behind when the -request fails, the header is missing, or its value is present but unparseable -(a rewriting proxy) — the last case keeps the JWT's last real correction rather -than handing `NaN` to `iat`. `clockSkewSeconds()` returns that stored -reading. The App JWT still refreshes the same figure at most every five minutes -and corrects `iat` with it; the JWT path is unchanged by this field. +unauthenticated `GET /rate_limit`, records it, and leaves the last successful +reading untouched when the request fails, the header is missing, or its value is +present but unparseable (a rewriting proxy) — an unparseable header used to hand +`NaN` to `iat`, and it never replaces a real reading now. `clockSkewSeconds()` +returns that stored reading and `clockState()` turns it into the state above. +The App JWT refreshes the same figure at most every five minutes and signs only +from a reading inside the 15-minute window, so a failed refresh can no longer +keep a stale correction signing. ## Conventions diff --git a/docs/architecture.md b/docs/architecture.md index 6d22b719..843f50b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -865,18 +865,27 @@ DSH token never reaches the browser. ### GitHub App onboarding and webhooks The optional GitHub App removes the PAT from the common path. Fledge generates a -short-lived App JWT (with a clock-skew correction from GitHub's `Date` header, -the same measurement reported by `/readyz` as `clock.skewSeconds` and by -`/metrics` as `fledge_clock_skew_seconds`; a failed measurement reports -`null` and no series at all rather than a zero that would read as aligned), -exchanges it for an installation token, lists the installation repositories and -stores only `repos.installation_id`; `repoGithubToken()` prefers a user OAuth token authorized with `repo` scope for human attribution and otherwise uses a fresh installation +short-lived App JWT, exchanges it for an installation token, lists the +installation repositories and stores only `repos.installation_id`; `repoGithubToken()` prefers a user OAuth token authorized with `repo` scope for human attribution and otherwise uses a fresh installation token when that column is set and falls back to the encrypted PAT otherwise. The installation callback binds the installation to the logged-in user (verified through the user's OAuth token or an individual account match). GitHub webhooks are verified with `X-Hub-Signature-256` before parsing; verified events are recorded in `events` for the follow-up loop. PAT onboarding stays available. +The JWT corrects `iat`/`exp` for the process clock's offset from GitHub's `Date` +header — the same measurement `/readyz` reports as `clock.skewSeconds` + +`clock.state` and `/metrics` as `fledge_clock_skew_seconds`. The offset is used, +not only surfaced (#758): a reading is `ok` for 15 minutes and inside 120 seconds, +`fail` above 120 seconds, and `unknown` when nothing was measured inside the +window. While it is `fail`, `/readyz` answers 503 and `Jobs.enqueue` refuses with +`CLOCK_SKEW` before any job row is written, so no run stores a time from a clock +nobody can trust; the JWT signs only in `ok` and otherwise raises +`CLOCK_SKEW_EXCEEDED` (`fail`) or `CLOCK_SKEW_UNVERIFIED` (`unknown`) instead of +signing from a stale or zero correction. A failed measurement reports `null` and +no series rather than a zero that would read as aligned, and it never replaces +the last reading that really was measured. + ### Inbound email replies When `INBOUND_EMAIL_SECRET` is set, the daily digest gets a diff --git a/docs/deploy.md b/docs/deploy.md index eb77c5be..6fa3ea84 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -1053,6 +1053,28 @@ merged, so a tag on `main` is evidence of a released version, not of a rollout. minutes. Fix the host clock (`timedatectl set-ntp true`, or start the container with a host that keeps time); Fledge reports the drift and never rewrites a stored timestamp to hide it. + + The measured offset is used, not only shown. `clock.state` says what the last + measurement means, and `clock.thresholdSeconds` carries the number it is + compared against — `CLOCK_SKEW_FAIL_SECONDS` in `src/repos/app.ts`, which must + stay equal to the `120` in `deploy/prometheus/fledge-alerts.yml`: + + | `clock.state` | meaning | `/readyz` | + |---|---|---| + | `not-configured` | no GitHub App on this deployment, so nothing is measured | `200` | + | `ok` | measured at most 15 minutes ago and within 120 s | `200` | + | `unknown` | never measured, or the last reading is older than 15 minutes | `200` | + | `fail` | the last measurement is outside 120 s | **`503`, `ok:false`** | + + In `fail` the platform refuses to start work: `Jobs.enqueue` answers + `CLOCK_SKEW` (counted as `fledge_admission_rejected_total{reason="clock_skew"}`) + before any job row is written, and the console banner names the measured + seconds. App JWTs are signed only in `ok`; `fail` raises `CLOCK_SKEW_EXCEEDED` + and `unknown` raises `CLOCK_SKEW_UNVERIFIED` instead of signing with a stale or + zero correction. A GitHub outage therefore stops App calls after at most 15 + minutes rather than serving a signature nobody re-measured. Fix the clock, then + confirm `/readyz` is `200` again — the next measurement (at most five minutes + after a `fail`) clears it. - The Admin panel shows the audit trail and recent jobs. - Quota state: `GET /v1/models` with a user token, or the dashboard. - Disk: SQLite WAL files and logs. diff --git a/src/app.ts b/src/app.ts index 07951d61..0563c07f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1031,10 +1031,21 @@ export async function buildApp(options: BuildOptions): Promise { const instances = store.listRepoInstances() const ready = instances.filter((entry) => entry.status === 'ready').length const failed = instances.filter((entry) => entry.status === 'failed').length - // #758: the skew the App JWT already corrects for, reported rather than - // re-measured. `null` means no measurement has succeeded — never 0, which - // would read as a clock that is exactly right. - return { ok: true, db: true, instances: { ready, failed, total: instances.length }, clock: { skewSeconds: githubApp.clockSkewSeconds(), source: 'github' } } + // #758 slice 2: readiness fails on a measured clock outside the threshold, so + // a monitor that reads only the status code stops seeing a "healthy" instance + // whose every stored time is 28 minutes ahead. `unknown` and + // `not-configured` stay 200: a GitHub outage, or a deployment with no App, is + // not a clock verdict. The object keeps `source` and adds `state` and + // `thresholdSeconds`, so a reader can tell "no number" from "a number that is + // too far". + const clock = githubApp.clockState() + const body = { + ok: clock.state !== 'fail', + db: true, + instances: { ready, failed, total: instances.length }, + clock: { ...clock, source: 'github' }, + } + return clock.state === 'fail' ? reply.code(503).send(body) : body }) app.get('/lang/:locale', async (request, reply) => { diff --git a/src/jobs/service.ts b/src/jobs/service.ts index b88b4f5b..8a8d80ec 100644 --- a/src/jobs/service.ts +++ b/src/jobs/service.ts @@ -1144,6 +1144,23 @@ export class Jobs { // implying this one repository is disabled, and it is checked in the admission // path itself: a due repository reaches `enqueue` without a route. if (this.platformStop() !== null) return { ok: false, error: '平台已紧急停止,暂不接受新任务。', code: 'PLATFORM_PAUSED' } + // #758 slice 2: a clock the last measurement puts outside the threshold must + // not start work that stores or schedules time. It sits after the platform + // stop (which outranks everything) and before the work-type gates, and it is + // here rather than on the routes because every trigger — manual, scheduled, + // follow-up, review, discovery — funnels through `enqueue`. Nothing is + // written: the refusal is before any job row, so no row carries a time from a + // clock nobody can trust. + const clock = this.githubApp.clockState() + if (clock.state === 'fail') { + incCounter('fledge_admission_rejected_total', { reason: 'clock_skew' }, 1, 'Queue admission rejections') + return { + ok: false, + error: `平台时钟与 GitHub 相差 ${String(clock.skewSeconds)} 秒,超过 ${String(clock.thresholdSeconds)} 秒上限,任务没有开始。`, + code: 'CLOCK_SKEW', + detail: String(clock.skewSeconds), + } + } // #390: the work type is resolved before the gates, and an unknown one is // refused. A default here would run work under a contract nobody declared. const workType = resolveWorkType(options.workType ?? workTypeForKind(kind)) diff --git a/src/lib/failure.ts b/src/lib/failure.ts index a506a9f0..5e78154b 100644 --- a/src/lib/failure.ts +++ b/src/lib/failure.ts @@ -204,6 +204,8 @@ export const ENQUEUE_REFUSAL_CODES = [ 'RUNNER_ERROR', 'RULES_UNAVAILABLE', 'RULES_INVALID', + // #758 slice 2: the platform clock is outside the threshold, so no job starts. + 'CLOCK_SKEW', ] as const export type EnqueueRefusalCode = (typeof ENQUEUE_REFUSAL_CODES)[number] @@ -262,6 +264,10 @@ export const REFUSAL_SENTENCES: Record = { // the file that crossed the limit and the byte count the maintainer can act on. RULES_UNAVAILABLE: ['无法读取这个仓库的规则或技能,任务没有开始。', "This repository's rules or skills could not be read, so the job did not start."], RULES_INVALID: ['这个仓库的规则或技能不合法:{detail}。', 'The repository rules or skills are invalid: {detail}.'], + // #758 slice 2: the refusal carries the measured seconds, because "the clock is + // wrong" without the number sends the reader to the wrong machine. The + // threshold rides on `clockState()` and in docs/deploy.md, not in this sentence. + CLOCK_SKEW: ['平台时钟与 GitHub 相差 {detail} 秒,超过 120 秒上限,任务没有开始。', 'The platform clock is {detail} seconds from GitHub time, above the 120-second threshold, so the job did not start.'], } const BRANCH_EMPTY: [string, string] = ['目标分支不存在或不可访问。', 'The target branch does not exist, or this account cannot read it.'] diff --git a/src/repos/app.ts b/src/repos/app.ts index e698d8bf..2738d950 100644 --- a/src/repos/app.ts +++ b/src/repos/app.ts @@ -41,15 +41,38 @@ function base64url(value: Buffer | string): string { const CLOCK_SKEW_LOG_SECONDS = 60 const SKEW_REFRESH_MS = 5 * 60_000 +// #758 slice 2: the number that turns a measurement into a decision. It is the +// same 120 the alert rule fires on (`deploy/prometheus/fledge-alerts.yml`) and +// `docs/deploy.md` names both, so a failing readiness probe and a firing alert +// cannot disagree about what "too far" means. +export const CLOCK_SKEW_FAIL_SECONDS = 120 + +// How long one successful measurement stays usable. Inside the window a GitHub +// outage is tolerated; past it the clock is unverified, which is not the same as +// good, and nothing may sign from a number nobody has re-measured. +export const CLOCK_SKEW_REUSE_MS = 15 * 60_000 + +export type ClockStateName = 'not-configured' | 'ok' | 'fail' | 'unknown' + +export interface ClockState { + /** The last successful measurement: seconds the process is ahead of GitHub. */ + skewSeconds: number | null + /** When that measurement was taken, ISO-8601, or null when none succeeded. */ + measuredAt: string | null + thresholdSeconds: number + state: ClockStateName +} + export class GithubApp { private key: string | null = null - private skewMs = 0 private skewAt = 0 - // The last successful measurement, kept apart from `skewMs`: the JWT needs a - // number, while a report has to be able to say "unknown". A 0 here would read - // as a clock that is exactly right, which is the one lie this slice exists to - // prevent. + // The last successful measurement. It is kept when a refresh fails — the + // reading is still what was measured, it just stops being trusted once + // `CLOCK_SKEW_REUSE_MS` passes — and it is the only number the JWT may sign + // with. `skewMs` used to be a private fail-open copy of it and is gone: that + // copy is how a host 28 minutes fast kept signing. private measuredSkewSeconds: number | null = null + private measuredAt: number | null = null private readonly tokens = new Map() constructor(private readonly config: Config) {} @@ -71,37 +94,34 @@ export class GithubApp { async measureSkew(): Promise { if (!this.configured) { this.measuredSkewSeconds = null + this.measuredAt = null return null } try { const response = await fetch(`${this.config.github.apiBase}/rate_limit`, { headers: { 'user-agent': 'fledge/0.1' }, signal: AbortSignal.timeout(10_000) }) const date = response.headers.get('date') - this.skewAt = Date.now() - if (date === null) { - this.measuredSkewSeconds = null - return null - } + const at = Date.now() + this.skewAt = at + if (date === null) return null const remote = Date.parse(date) if (!Number.isFinite(remote)) { // #758 finding: a `Date` header that is present but unparseable (a // rewriting proxy is the realistic case) is a failed measurement, not a - // reading. `skewMs` keeps the last real correction for the JWT, and the - // report says "unknown" — `NaN` here used to travel to /readyz, to the - // metric, and into the JWT's `iat` without failing anything. - this.measuredSkewSeconds = null + // reading — `NaN` here used to travel to /readyz, to the metric, and into + // the JWT's `iat` without failing anything. The previous reading stays; + // it ages out of `ok` on its own rather than being replaced by a guess. return null } - this.skewMs = Date.now() - remote - const seconds = Math.round(this.skewMs / 1000) + const seconds = Math.round((at - remote) / 1000) this.measuredSkewSeconds = seconds + this.measuredAt = at if (Math.abs(seconds) > CLOCK_SKEW_LOG_SECONDS) log.warn('clock.skew', { skewSeconds: seconds, source: 'github' }) return seconds } catch { - // The JWT keeps its last correction here; slice 2 of #758 owns that hole. - // The report does not keep it, because a measurement that failed is not a - // reading, and a stale number would hide the next drift. + // A failed refresh is not a reading: the last successful one stays as the + // report's answer and ages out of `ok` after `CLOCK_SKEW_REUSE_MS`. The JWT + // has no correction of its own to fall back on any more. this.skewAt = Date.now() - this.measuredSkewSeconds = null return null } } @@ -111,10 +131,38 @@ export class GithubApp { return this.measuredSkewSeconds } + // #758 slice 2: one answer to "may anything act on this clock", read by + // /readyz, by `appJwt` and by `Jobs.enqueue`. `not-configured` is a deployment + // with no App, which is not a clock verdict; `unknown` is "nobody measured, or + // the last reading is stale"; `fail` is a measurement above the threshold, and + // it stays `fail` once it ages out — an outage does not repair a clock that was + // measured 28 minutes wrong. + clockState(): ClockState { + const thresholdSeconds = CLOCK_SKEW_FAIL_SECONDS + if (!this.configured) return { skewSeconds: null, measuredAt: null, thresholdSeconds, state: 'not-configured' } + const seconds = this.measuredSkewSeconds + const at = this.measuredAt + if (seconds === null || at === null) return { skewSeconds: null, measuredAt: null, thresholdSeconds, state: 'unknown' } + const measuredAt = new Date(at).toISOString() + if (Math.abs(seconds) > thresholdSeconds) return { skewSeconds: seconds, measuredAt, thresholdSeconds, state: 'fail' } + if (Date.now() - at > CLOCK_SKEW_REUSE_MS) return { skewSeconds: seconds, measuredAt, thresholdSeconds, state: 'unknown' } + return { skewSeconds: seconds, measuredAt, thresholdSeconds, state: 'ok' } + } + private async appJwt(): Promise { if (!this.configured) throw new Error('GitHub App 未配置') if (Date.now() - this.skewAt > SKEW_REFRESH_MS) await this.measureSkew() - const now = Math.floor((Date.now() - this.skewMs) / 1000) + // #758 slice 2: sign only with a measurement that is inside both the reuse + // window and the threshold. Everything else refuses with a named error + // instead of signing from a stale or zero correction — that fail-open is what + // let a host 28 minutes fast keep talking to GitHub as if its clock were + // right, and a 401 from GitHub was the only symptom. + const clock = this.clockState() + if (clock.state === 'fail') { + throw new Error(`CLOCK_SKEW_EXCEEDED: the process clock is ${String(clock.skewSeconds)}s from GitHub time, above the ${String(clock.thresholdSeconds)}s threshold`) + } + if (clock.state !== 'ok') throw new Error('CLOCK_SKEW_UNVERIFIED: no clock measurement inside the reuse window') + const now = Math.floor(Date.now() / 1000) - (clock.skewSeconds ?? 0) const header = base64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' })) const payload = base64url(JSON.stringify({ iat: now - 60, exp: now + 540, iss: this.config.githubApp.id })) const signer = createSign('RSA-SHA256') diff --git a/test/clockSkew.test.ts b/test/clockSkew.test.ts index 51fa0a16..53397dd1 100644 --- a/test/clockSkew.test.ts +++ b/test/clockSkew.test.ts @@ -216,9 +216,9 @@ test('/readyz reads the last measurement instead of calling GitHub', async () => // through the whole chain: `skewSeconds: NaN` on /readyz (serialised as `null` by // luck), a literal `NaN` gauge on /metrics because `setGauge` never checked // finiteness, a `Math.abs(NaN) > 60` log test that is false, and an `iat: NaN` -// JWT. A number that was never measured, failing silently. The fourth way to be -// unknown is the same as the other three: no reading, no series. -test('a Date header that is present but unparseable is no measurement, and the JWT keeps its last real correction', async () => { +// JWT. A number that was never measured, failing silently. An unparseable header +// is therefore a failed measurement: it may not replace the last real reading. +test('a Date header that is present but unparseable is no measurement, so the last real reading stands', async () => { resetMetrics() const dir = mkdtempSync(join(tmpdir(), 'fledge-clock-skew-')) const keyPath = join(dir, 'app.pem') @@ -233,7 +233,7 @@ test('a Date header that is present but unparseable is no measurement, and the J const realNow = Date.now const measuredAt = 1_800_000_000_000 try { - // A real reading first. This is the last real correction the JWT may keep. + // A real reading first. This is the reading the failure below must not move. Date.now = () => measuredAt assert.equal(await h.built.githubApp.measureSkew(), 90, 'the fake clock is 90 seconds behind') @@ -243,14 +243,20 @@ test('a Date header that is present but unparseable is no measurement, and the J const loggedBefore = capture.lines.length assert.equal(await h.built.githubApp.measureSkew(), null, 'an unparseable Date was recorded as a measurement') + // #758 slice 2: the failed refresh keeps the last successful reading, which is + // 2.5 s old and inside the reuse window, so the report still names the number + // that was really measured — with its state, so a reader can tell it apart. const ready = await h.built.app.inject({ method: 'GET', url: '/readyz' }) - const body = ready.json() as { clock?: { skewSeconds?: number | null } } - assert.equal(body.clock?.skewSeconds, null, 'an unparseable Date read as a number on /readyz') + const body = ready.json() as { clock?: { skewSeconds?: number | null; state?: string } } + assert.equal(ready.statusCode, 200) + assert.equal(body.clock?.skewSeconds, 90, 'an unparseable Date replaced the last real reading') + assert.equal(body.clock?.state, 'ok') const metrics = await h.built.app.inject({ method: 'GET', url: '/metrics', remoteAddress: '127.0.0.1' }) - assert.doesNotMatch(metrics.body, /fledge_clock_skew_seconds/, 'an unparseable Date published a gauge') + assert.match(metrics.body, /^fledge_clock_skew_seconds 90$/m, 'an unparseable Date published a NaN gauge') assert.deepEqual(capture.lines.slice(loggedBefore).filter((line) => line.msg === 'clock.skew'), [], 'a NaN skew passed the 60-second log test') - // The JWT still mints, with the last correction that was really measured. + // The JWT still mints, with the same real correction: a failed refresh inside + // the window is not a clock verdict. // `floor` truncates the half-second toward the past; that is the asserted // direction, not a tolerance around the nearest second. await h.built.githubApp.getApp() diff --git a/test/clockSkewFail.test.ts b/test/clockSkewFail.test.ts new file mode 100644 index 00000000..c9a1687e --- /dev/null +++ b/test/clockSkewFail.test.ts @@ -0,0 +1,230 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { generateKeyPairSync } from 'node:crypto' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { buildApp } from '../src/app.js' +import { Store } from '../src/db.js' +import { MockRunner } from '../src/jobs/mock.js' +import { resetMetrics } from '../src/lib/metrics.js' +import { json, makeUser, readBody, startFakeServer, testConfig } from './helpers.js' + +// #758 slice 2: the measured offset is used, not only reported. Slice 1 made the +// skew visible on /readyz and /metrics; this file pins the decision that follows +// from it — readiness fails above 120 s, `enqueue` refuses before writing a row, +// and the App JWT signs only from a measurement inside the reuse window. The host +// clock cannot be fixed by a commit, so the platform's answer is to stop acting +// on a clock it cannot trust. + +const keyDir = mkdtempSync(join(tmpdir(), 'fledge-clock-fail-')) +const keyPath = join(keyDir, 'app.pem') +const { privateKey } = generateKeyPairSync('rsa', { + modulusLength: 2048, + privateKeyEncoding: { type: 'pkcs1', format: 'pem' }, + publicKeyEncoding: { type: 'spki', format: 'pem' }, +}) +writeFileSync(keyPath, privateKey) + +interface Options { + /** Seconds the faked GitHub clock is behind, i.e. how far ahead the process is. */ + aheadSeconds?: number + /** Skip the fake server; the App is configured but GitHub is unreachable. */ + unreachable?: boolean + /** No App on this deployment, as on a self-hosted instance without one. */ + unconfigured?: boolean +} + +async function harness(options: Options = {}) { + const calls: string[] = [] + const server = options.unreachable === true + ? null + : await startFakeServer(async (request, response) => { + await readBody(request) + calls.push(request.url ?? '') + const date = new Date(Date.now() - (options.aheadSeconds ?? 5) * 1000).toUTCString() + if ((request.url ?? '').endsWith('/app')) return json(response, 200, { id: 123456, slug: 'fledge-clock', name: 'fledge-clock' }, { date }) + return json(response, 200, { resources: {} }, { date }) + }) + const base = testConfig() + const config = testConfig({ + github: { ...base.github, apiBase: server === null ? 'http://127.0.0.1:9' : server.url }, + githubApp: { + id: options.unconfigured === true ? '' : '123456', + slug: 'fledge-clock', + privateKeyPath: options.unconfigured === true ? '' : keyPath, + webhookSecret: '', + useUserToken: false, + }, + }) + const store = new Store(':memory:') + const built = await buildApp({ config, store, mailer: undefined, runner: new MockRunner(), startScheduler: false }) + const user = makeUser(store, { login: 'clock-skew-user', status: 'approved' }) + const repo = store.createRepo({ + id: 'repo_clock', + userId: user.id, + owner: 'acme', + name: 'clock', + defaultBranch: 'main', + tokenEnc: 'x', + permission: 'admin', + repoRole: 'maintainer', + goal: 'ship it', + autoIterate: false, + intervalMinutes: 1440, + }) + return { + built, + store, + user, + repo, + calls, + stopGithub: async () => { + if (server !== null) await server.close() + }, + close: async () => { + await built.close() + if (server !== null) await server.close() + }, + } +} + +test('a clock outside the threshold fails readiness, refuses work and refuses to sign', async () => { + resetMetrics() + const h = await harness({ aheadSeconds: 28 * 60 }) + try { + const measured = await h.built.githubApp.measureSkew() + assert.ok(measured !== null && Math.abs(measured - 1680) <= 2, `measured ${String(measured)}`) + const state = h.built.githubApp.clockState() + assert.equal(state.state, 'fail') + assert.equal(state.thresholdSeconds, 120) + assert.ok(typeof state.skewSeconds === 'number' && Math.abs(state.skewSeconds - 1680) <= 2) + assert.ok(state.measuredAt !== null && !Number.isNaN(Date.parse(state.measuredAt))) + + const ready = await h.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(ready.statusCode, 503, '/readyz stayed 200 on a clock 28 minutes ahead') + const body = ready.json() as { ok?: boolean; clock?: { state?: string; thresholdSeconds?: number; skewSeconds?: number | null; source?: string } } + assert.equal(body.ok, false) + assert.equal(body.clock?.state, 'fail') + assert.equal(body.clock?.thresholdSeconds, 120) + assert.equal(body.clock?.source, 'github') + assert.ok(typeof body.clock?.skewSeconds === 'number' && Math.abs(body.clock.skewSeconds - 1680) <= 2) + + const refused = await h.built.jobs.enqueue(h.repo, 'manual', 'ship it') + assert.equal(refused.ok, false) + assert.equal(refused.code, 'CLOCK_SKEW') + assert.match(String(refused.detail), /^16\d\d$/) + assert.equal(h.store.listJobsForRepo(h.repo.id, 50).length, 0, 'a job row was written under a bad clock') + + const callsBefore = h.calls.length + await assert.rejects(async () => { await h.built.githubApp.getApp() }, /CLOCK_SKEW_EXCEEDED/) + assert.equal(h.calls.length, callsBefore, 'a JWT was minted from a clock above the threshold') + } finally { + await h.close() + } +}) + +test('a clock inside the threshold keeps readiness and work green', async () => { + resetMetrics() + const h = await harness({ aheadSeconds: 5 }) + try { + const measured = await h.built.githubApp.measureSkew() + assert.ok(measured !== null && Math.abs(measured - 5) <= 1, `measured ${String(measured)}`) + assert.equal(h.built.githubApp.clockState().state, 'ok') + + const ready = await h.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(ready.statusCode, 200) + const body = ready.json() as { ok?: boolean; clock?: { state?: string } } + assert.equal(body.ok, true) + assert.equal(body.clock?.state, 'ok') + + const accepted = await h.built.jobs.enqueue(h.repo, 'manual', 'ship it') + assert.equal(accepted.ok, true, `enqueue refused a good clock: ${String(accepted.code)}`) + assert.equal(h.store.listJobsForRepo(h.repo.id, 50).length, 1) + } finally { + await h.close() + } +}) + +test('120 seconds is inside the threshold and 121 is outside it', async () => { + resetMetrics() + const inside = await harness({ aheadSeconds: 120 }) + const outside = await harness({ aheadSeconds: 121 }) + // `Date` headers carry whole seconds, so an unfrozen clock measures 120.0-120.9 s + // at this boundary and rounds either way; the assertion is about the comparison, + // not about network latency, so both readings are taken at one frozen instant. + const realNow = Date.now + const measuredAt = 1_800_000_000_000 + try { + Date.now = () => measuredAt + assert.equal(await inside.built.githubApp.measureSkew(), 120) + assert.equal(inside.built.githubApp.clockState().state, 'ok') + const ready = await inside.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(ready.statusCode, 200, '120 s is the threshold, not past it') + assert.equal((ready.json() as { clock?: { state?: string } }).clock?.state, 'ok') + + assert.equal(await outside.built.githubApp.measureSkew(), 121) + assert.equal(outside.built.githubApp.clockState().state, 'fail') + const refused = await outside.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(refused.statusCode, 503) + assert.equal((refused.json() as { clock?: { state?: string } }).clock?.state, 'fail') + } finally { + Date.now = realNow + await inside.close() + await outside.close() + } +}) + +test('a failed refresh keeps the last reading for fifteen minutes, then turns unverified', async () => { + resetMetrics() + const h = await harness({ aheadSeconds: 5 }) + const realNow = Date.now + const measuredAt = 1_800_000_000_000 + try { + Date.now = () => measuredAt + assert.equal(await h.built.githubApp.measureSkew(), 5) + + // GitHub stops answering. A refresh failure is not a reading, and the last + // successful one is still inside the window, so the clock stays usable. + await h.stopGithub() + Date.now = () => measuredAt + 14 * 60_000 + assert.equal(await h.built.githubApp.measureSkew(), null) + assert.equal(h.built.githubApp.clockState().state, 'ok', 'a failed refresh inside the window stopped trusting the reading') + const ready = await h.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(ready.statusCode, 200, 'an outage was read as a clock verdict') + + // Past the window the reading is no longer evidence: unknown, not "fine". + Date.now = () => measuredAt + 16 * 60_000 + assert.equal(h.built.githubApp.clockState().state, 'unknown') + const stale = await h.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(stale.statusCode, 200) + assert.equal((stale.json() as { clock?: { state?: string } }).clock?.state, 'unknown') + await assert.rejects(async () => { await h.built.githubApp.getApp() }, /CLOCK_SKEW_UNVERIFIED/) + } finally { + Date.now = realNow + await h.close() + } +}) + +test('a deployment with no App is not a clock verdict', async () => { + resetMetrics() + const h = await harness({ unconfigured: true }) + try { + assert.equal(await h.built.githubApp.measureSkew(), null) + const state = h.built.githubApp.clockState() + assert.equal(state.state, 'not-configured') + assert.equal(state.skewSeconds, null) + + const ready = await h.built.app.inject({ method: 'GET', url: '/readyz' }) + assert.equal(ready.statusCode, 200, 'a deployment without an App was failed for a clock it never measures') + const body = ready.json() as { ok?: boolean; clock?: { state?: string; skewSeconds?: number | null } } + assert.equal(body.ok, true) + assert.equal(body.clock?.state, 'not-configured') + assert.equal(body.clock?.skewSeconds, null) + + const accepted = await h.built.jobs.enqueue(h.repo, 'manual', 'ship it') + assert.equal(accepted.ok, true, `enqueue refused a deployment that has no App: ${String(accepted.code)}`) + } finally { + await h.close() + } +})