diff --git a/.github/workflows/rerun-infra-flakes.yml b/.github/workflows/rerun-infra-flakes.yml index 1cb1668c8..29106867c 100644 --- a/.github/workflows/rerun-infra-flakes.yml +++ b/.github/workflows/rerun-infra-flakes.yml @@ -24,8 +24,9 @@ jobs: RUN_ID: ${{ github.event.workflow_run.id }} run: | set -euo pipefail - # Signatures seen on real runs: setup-bazel's release lookup, a stale apt index, runner teardown. - infra='Unable to find Bazelisk version|E: Failed to fetch|E: Unable to fetch some archives|The runner has received a shutdown signal|The hosted runner lost communication' + # Signatures seen on real runs: setup-bazel's release lookup, a stale apt index, runner teardown, a Maven registry download (or a + # Coursier artifact fetch) that failed during a Bazel repository fetch (not a missing artifact: those are 404 warnings, which Bazel falls through). + infra='Unable to find Bazelisk version|E: Failed to fetch|E: Unable to fetch some archives|The runner has received a shutdown signal|The hosted runner lost communication|java.io.IOException: Error downloading \[https://(repo1.maven.org|maven.google.com|jitpack.io|maven-central.storage-download.googleapis.com)|Error while fetching artifact with coursier' failed=$(gh api "repos/$REPO/actions/runs/$RUN_ID/jobs?filter=latest&per_page=100" \ --jq '.jobs[] | select(.conclusion == "failure") | [.id, .name, ([.steps[] | select(.conclusion == "failure")] | length)] | @tsv') diff --git a/kmp/src/jvmTest/kotlin/dev/stapler/stelekit/sketch/v2/SketchEngineBenchmark.kt b/kmp/src/jvmTest/kotlin/dev/stapler/stelekit/sketch/v2/SketchEngineBenchmark.kt index 7c9374f16..898328385 100644 --- a/kmp/src/jvmTest/kotlin/dev/stapler/stelekit/sketch/v2/SketchEngineBenchmark.kt +++ b/kmp/src/jvmTest/kotlin/dev/stapler/stelekit/sketch/v2/SketchEngineBenchmark.kt @@ -90,6 +90,17 @@ class SketchEngineBenchmark { return (0 until rounds).minOf { val t0 = System.nanoTime(); block(); (System.nanoTime() - t0) / NANOS_PER_MS } } + /** + * [bestOfMs], but a figure over [limitMs] is re-measured up to [CONFIRMATIONS] more times and the lowest wins. A shared CI + * runner can be starved for a whole best-of-N window (a 7 ms solve read 58 ms); a real regression stays over the limit + * in every attempt, so the gate still fails on it. + */ + private fun confirmedBestOfMs(limitMs: Double, rounds: Int = ROUNDS, warmups: Int = WARMUPS, block: () -> Unit): Double { + var best = bestOfMs(rounds, warmups, block) + repeat(CONFIRMATIONS) { if (best > limitMs) best = minOf(best, bestOfMs(rounds, warmups, block)) } + return best + } + private fun report(key: String, figure: Double, extra: String = "") { println("SKETCH-BENCH {\"key\":\"$key\",\"value\":$figure$extra}") } @@ -103,20 +114,28 @@ class SketchEngineBenchmark { fun frame500_should_StayWithinTheRatioToPrebuiltPaths_AndTheCeiling() { val scene = (0 until FRAME_ELEMENTS).fold(SketchSceneV2.empty(limits)) { s, i -> s.add(element(i, FRAME_COLUMNS)).getOrNull()!! } val ops = DrawListBuilder(metrics).build(scene, RenderStyle.Clean) - val painter = ScenePainter() - val bitmap = ImageBitmap(FRAME_WIDTH, FRAME_HEIGHT, ImageBitmapConfig.Argb8888) - val painted = ArrayList() - for (frame in 0 until WARMUP_FRAMES + FRAMES) { - val t0 = System.nanoTime() - drawTo(bitmap) { painter.paint(this, ops, SketchViewport.of(panX = -frame * PAN_X, panY = -frame * PAN_Y), null) } - if (frame >= WARMUP_FRAMES) painted += (System.nanoTime() - t0) / NANOS_PER_MS - } - val reference = prebuiltPathFrames() - val painterP95 = p95(painted) - val referenceP95 = p95(reference) val budget = entry("sketch.frame_ms_500_elements") val maxRatio = budget.num("max_ratio_to_prebuilt_paths") val ceiling = budget.num("ceiling_ms") + fun measure(): Pair { + val painter = ScenePainter() + val bitmap = ImageBitmap(FRAME_WIDTH, FRAME_HEIGHT, ImageBitmapConfig.Argb8888) + val painted = ArrayList() + for (frame in 0 until WARMUP_FRAMES + FRAMES) { + val t0 = System.nanoTime() + drawTo(bitmap) { painter.paint(this, ops, SketchViewport.of(panX = -frame * PAN_X, panY = -frame * PAN_Y), null) } + if (frame >= WARMUP_FRAMES) painted += (System.nanoTime() - t0) / NANOS_PER_MS + } + return p95(painted) to p95(prebuiltPathFrames()) + } + // Frames are 1-5 ms, so one starved window moves the ratio past the limit; a real regression is over it in every attempt. + var (painterP95, referenceP95) = measure() + repeat(CONFIRMATIONS) { + if (painterP95 > referenceP95 * maxRatio || painterP95 > ceiling) { + val (p, r) = measure() + if (p / r < painterP95 / referenceP95) { painterP95 = p; referenceP95 = r } + } + } report("sketch.frame_ms_500_elements", painterP95, ",\"referenceP95\":$referenceP95,\"ratio\":${painterP95 / referenceP95}") assertTrue(painterP95 <= referenceP95 * maxRatio, "painter p95 $painterP95 ms against $referenceP95 ms of prebuilt paths (x$maxRatio)") assertTrue(painterP95 <= ceiling, "painter p95 $painterP95 ms is over the $ceiling ms ceiling") @@ -156,13 +175,14 @@ class SketchEngineBenchmark { @Test fun solve100Constraints_should_StayUnderTheBudget() { val solver = Solver() + val limit = entry("sketch.solve_ms_100_constraints").num("max") var worst = 0.0 for (seed in SolverCorpus.corpusSeeds) { val case = SolverCorpus.hundredConstraints(seed) - worst = maxOf(worst, bestOfMs(rounds = SOLVE_ROUNDS, warmups = 1) { solver.solve(case.problem, case.start) }) + worst = maxOf(worst, confirmedBestOfMs(limit, rounds = SOLVE_ROUNDS, warmups = 1) { solver.solve(case.problem, case.start) }) } report("sketch.solve_ms_100_constraints", worst) - assertTrue(worst <= entry("sketch.solve_ms_100_constraints").num("max"), "worst seed $worst ms") + assertTrue(worst <= limit, "worst seed $worst ms") } @Test @@ -260,6 +280,7 @@ class SketchEngineBenchmark { const val ROUNDS = 5 const val WARMUPS = 2 const val SOLVE_ROUNDS = 5 + const val CONFIRMATIONS = 2 const val IO_ROUNDS = 5 const val HIT_PROBES = 1_000 const val PROBE_STRIDE = 37