diff --git a/ci/README.md b/ci/README.md index 68b0c6e..cd90231 100644 --- a/ci/README.md +++ b/ci/README.md @@ -82,12 +82,20 @@ every sync it checks: Nothing stops a run early: every round runs, and each check is a row in the job summary. The JSON results are kept as an artifact for 30 days. +A check is **skipped** (⏭️), not failed, when the Fetch API never showed a change it depends on, so it +can't grade the CLI: a wait for the Fetch API that times out, a later `drift:listOrder` on a list whose +earlier change never showed, and a `verify` whose only differences are list orders the CLI's own +preflight reports as "not in the Fetch API yet". Skips are counted in the summary and never fail the +run. (On dev, a list reorder or redirection edit saved while that instance's push holds its lease can +be dropped by Repo Push and never reach the Fetch API.) + ## Promotion to `latest` A `v*` tag push runs this test on the tag's build first, and `npm-publish.yml` publishes to npm (under `beta`) only if it passed, so every version on npm from then on has passed it. If it failed -for a reason unrelated to the CLI, re-run the failed jobs of that tag's _Publish to npm_ run: the -test runs again on fresh instances, then the publish. Promote a published version to `latest` with +for a reason unrelated to the CLI, re-run **all jobs** of that tag's _Publish to npm_ run: the test +runs again on fresh instances, then the publish. (Re-running only the failed jobs reuses the leases +the first attempt already released, and fails.) Promote a published version to `latest` with _Publish to npm_ → _Run workflow_. ## Cleanup diff --git a/ci/permutation/run.mjs b/ci/permutation/run.mjs index 70fa504..e24364f 100644 --- a/ci/permutation/run.mjs +++ b/ci/permutation/run.mjs @@ -25,6 +25,12 @@ // Syncs and preflights pass --syncDeletes, so a delete on one side reaches the other. // Nothing stops the run early: every round runs and every result is recorded. // +// A check is skipped, not failed, when the Fetch API never showed a change it depends on: the +// environment didn't deliver what the sync reads, so the check can't grade the CLI. That covers a +// wait for the Fetch API that times out, a later drift:listOrder on a list whose earlier change never +// showed, and a verify whose only differences are list orders the CLI itself reports as not in the +// Fetch API yet. A skip never fails the run. +// // Env: SOURCE_GUID, TARGET_GUID, AGILITY_TOKEN, CLI (command for the CLI under test), // TIER (quick|beta|nightly), SEED (default random, printed), LOCALE (default en-us), // OUT_DIR (default ./permutation-out). Exit 0 when every check passes, 1 otherwise. @@ -81,11 +87,28 @@ function cli(step, args) { return { exitCode: p.status ?? -1, json, seconds: Math.round((Date.now() - started) / 1000) }; } +// `pass` is true, false, or SKIP (the environment, not the CLI; see the header) +const SKIP = "skip"; +const icon = (pass) => (pass === SKIP ? "⏭️" : pass ? "✅" : "❌"); function record(round, direction, check, pass, detail, seconds = 0) { results.push({ round, direction, check, pass, detail, seconds }); - console.log(`${pass ? "✅" : "❌"} [r${round} ${direction}] ${check}: ${detail}`); + console.log(`${icon(pass)} [r${round} ${direction}] ${check}: ${detail}`); +} + +// A wait for the Fetch API that ran out of time (fixtures.mjs throws these): a skip, not a failure. +// Any other error is still a failure. +const FETCH_TIMEOUT = /in the Fetch API after \d+s|Fetch API still (behind|shows the old order)/; +function recordApplyError(round, direction, check, e) { + const message = String(e?.message ?? e).slice(0, 200); + const lagged = FETCH_TIMEOUT.test(message); + record(round, direction, check, lagged ? SKIP : false, lagged ? `environment: ${message}` : message); + return lagged; } +// Lists (lowercased reference names) with a change the Fetch API never showed. The CLI rightly treats +// such a list as pending until it does, so its later list-order checks can't grade the CLI. +const fetchStale = new Set(); + // Content ID on the other instance, from the sync's own mapping file (one file serves both directions). function mapped(fromSourceSide) { const file = join(WORK, "agility-files", "mappings", `${A}-${B}`, LOCALE, "item", "mappings.json"); @@ -171,7 +194,28 @@ async function checkStep(round, direction) { { 0: "match", 1: `${findings.length || "some"} differences${kinds ? ` (${kinds})` : ""}`, 2: "could not verify" }[ v.exitCode ] ?? `exit ${v.exitCode}`; - record(round, direction, "verify", v.exitCode === 0, `${outcome}${later}`, v.seconds); + // Lists the no-op preflight holds back because the last sync's change isn't in the Fetch API yet. + // A verify whose only differences are those lists' order is the environment, not the CLI. + const pending = new Set( + (n.json?.preflight?.entries ?? []) + .filter((e) => e.action === "skip" && /isn't in the Fetch API yet/.test(e.detail ?? "")) + .map((e) => String(e.name).toLowerCase()) + ); + const onlyPending = + v.exitCode === 1 && + findings.length > 0 && + findings.every((x) => x.category === "listOrderMismatch" && pending.has(String(x.name).toLowerCase())); + if (onlyPending) for (const x of findings) fetchStale.add(String(x.name).toLowerCase()); + record( + round, + direction, + "verify", + v.exitCode === 0 || (onlyPending ? SKIP : false), + onlyPending + ? `environment: ${outcome}, all in lists the CLI reports as not in the Fetch API yet${later}` + : `${outcome}${later}`, + v.seconds + ); record( round, direction, @@ -298,7 +342,7 @@ for (let i = 0; i < plan.length; i++) { applied.push({ name: `drift:${d.kind}`, ...did, area: "drift" }); } else console.log(` (skipped drift:${d.kind}: nothing it applies to)`); } catch (e) { - record(round, direction, `apply:drift:${d.kind}`, false, String(e?.message ?? e).slice(0, 200)); + recordApplyError(round, direction, `apply:drift:${d.kind}`, e); } } @@ -310,7 +354,8 @@ for (let i = 0; i < plan.length; i++) { applied.push({ name: "drift:urlRedirection", ...did, area: "drift" }); } else record(round, direction, "apply:drift:urlRedirection", false, `${redirection.originUrl} isn't on ${to}`); } catch (e) { - record(round, direction, "apply:drift:urlRedirection", false, String(e?.message ?? e).slice(0, 200)); + // The edit was saved but never showed: no drift:urlRedirection check this round + recordApplyError(round, direction, "apply:drift:urlRedirection", e); } } @@ -335,7 +380,7 @@ for (let i = 0; i < plan.length; i++) { const waited = await waitForFetchVersions(AGILITY_TOKEN, guid, LOCALE, changed); if (changed.size) console.log(` Fetch API showed ${changed.size} edited item(s) on ${guid} after ${waited}s`); } catch (e) { - record(round, direction, "apply:fetch-catch-up", false, String(e?.message ?? e).slice(0, 200)); + recordApplyError(round, direction, "apply:fetch-catch-up", e); } } // A list swap's new order shows in the Fetch API later than the items' new versions @@ -344,7 +389,7 @@ for (let i = 0; i < plan.length; i++) { const waited = await waitForFetchItemOrder(AGILITY_TOKEN, a.orders.guid, LOCALE, a.orders.items); console.log(` Fetch API showed the new order of ${a.entity ?? a.detail} on ${a.orders.guid} after ${waited}s`); } catch (e) { - record(round, direction, "apply:fetch-order", false, String(e?.message ?? e).slice(0, 200)); + if (recordApplyError(round, direction, "apply:fetch-order", e) && a.entity) fetchStale.add(a.entity.toLowerCase()); } } @@ -395,16 +440,19 @@ for (let i = 0; i < plan.length; i++) { for (const d of structural) { const warned = keptByType(d.kind, d.entity); const conflicted = conflictIn(d.phase, d.entity); + const stale = d.kind === "listOrder" && fetchStale.has(String(d.entity).toLowerCase()); record( round, direction, `drift:${d.kind}`, - warned || conflicted, + warned || conflicted || (stale ? SKIP : false), warned ? `${d.entity}: kept, with a target-changed warning` : conflicted ? `${d.entity}: reported as a conflict (the round's own changes touched it too)` - : `${d.entity}: the other side's edit was overwritten without --overwrite, or not seen` + : stale + ? `${d.entity}: environment: an earlier change to this list never showed in the Fetch API, so the CLI treats it as pending` + : `${d.entity}: the other side's edit was overwritten without --overwrite, or not seen` ); kept ||= warned || conflicted; } @@ -425,11 +473,12 @@ writeFileSync( JSON.stringify({ cliVersion, tier: TIER, seed: SEED, source: A, target: B, rounds, results }, null, 2) ); -const failed = results.filter((x) => !x.pass); +const failed = results.filter((x) => x.pass === false); +const skipped = results.filter((x) => x.pass === SKIP); const md = [ `### Sync permutation test: CLI ${cliVersion}, tier ${TIER}, seed ${SEED}`, "", - `A \`${A}\` ↔ B \`${B}\`: ${results.length} checks, **${failed.length} failed**. Replay with \`SEED=${SEED} TIER=${TIER}\`.`, + `A \`${A}\` ↔ B \`${B}\`: ${results.length} checks, **${failed.length} failed**${skipped.length ? `, ${skipped.length} skipped (the Fetch API never showed a change)` : ""}. Replay with \`SEED=${SEED} TIER=${TIER}\`.`, "", "| Round | Direction | Changes |", "|---|---|---|", @@ -439,7 +488,7 @@ const md = [ "|---|---|---|---|---|---|", ...results.map( (x) => - `| ${x.round} | ${x.direction} | ${x.check} | ${x.pass ? "✅" : "❌"} | ${String(x.detail).replace(/\|/g, "\\|")} | ${x.seconds}s |` + `| ${x.round} | ${x.direction} | ${x.check} | ${icon(x.pass)} |${String(x.detail).replace(/\|/g, "\\|")} | ${x.seconds}s |` ), "", ].join("\n");