From 449a2b6bddc2528e3f5eff6f05d736781b6f6129 Mon Sep 17 00:00:00 2001 From: Steven Cady Date: Thu, 25 Jun 2026 18:34:01 -0500 Subject: [PATCH 1/2] ci(#17): code coverage measurement + ratchet gate in build.ps1; baseline + CI artifact Co-Authored-By: Claude Opus 4.8 --- .github/CONTRIBUTING.md | 32 +++++++++- .github/workflows/ci.yml | 12 +++- .gitignore | 2 + build.ps1 | 123 +++++++++++++++++++++++++++++++++++++-- coverage-baseline.json | 22 +++++++ 5 files changed, 183 insertions(+), 8 deletions(-) create mode 100644 coverage-baseline.json diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 24ae48e..795dce4 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -158,10 +158,38 @@ Invoke-Pester -Path ./Tests/ConfigManager.Tests.ps1 -Output Detailed `windows-latest`); run it locally to reproduce CI exactly: ```powershell -./build.ps1 # PSScriptAnalyzer (Error+ParseError, baselined) + Pester (excludes E2E/Integration) -./build.ps1 -InstallDeps # also installs the pinned PSToml / Pester / PSScriptAnalyzer versions +./build.ps1 # analyzer (Error+ParseError, baselined) + Pester (excludes E2E/Integration) +./build.ps1 -InstallDeps # also installs the pinned PSToml / Pester / PSScriptAnalyzer versions +./build.ps1 -Coverage # also measures coverage and enforces the ratchet gate (below) +./build.ps1 -InstallDeps -Coverage # exactly what CI runs ``` +### Code coverage gate and ratchet + +CI runs `build.ps1 -Coverage`, which folds JaCoCo code coverage over `Modules/` into the **same** +Pester run (no second pass) and writes `coverage.xml` (gitignored; uploaded as the `coverage-report` +CI artifact next to `pester-results`/`testResults.xml`). Coverage is **opt-in** via `-Coverage` +because the breakpoint instrumentation adds roughly 30% wall-clock to the suite; a plain +`./build.ps1` quick run skips it, and the gate only applies when coverage was actually measured. + +The gate is a **ratchet**: coverage can hold or climb, never silently fall. + +- The committed baseline lives in **`coverage-baseline.json`** (`{ "overall": , "perModule": { ... } }`). +- The build **FAILS** if the measured overall coverage drops more than the tolerance (0.75 pts, to + absorb run-to-run jitter from timing-dependent line hits) below `overall`. On a drop it prints + `coverage X% is below baseline Y%` and names the modules that regressed. +- When coverage rises more than ~1 pt above the baseline, the build prints a + `ratchet: baseline can be raised to X%` hint. It does **not** auto-raise — see below. + +**Raising the baseline (do this when you add tests that lift coverage):** + +1. Run `./build.ps1 -Coverage` and note the printed overall % (and the ratchet hint). +2. Edit `coverage-baseline.json`: set `overall` to the new number, and update the relevant + `perModule` entries (the per-module values are the module command hit-ratios the gate prints). +3. Commit it in the same PR. Raising the floor is deliberate so the new minimum is reviewed. + +Never *lower* the baseline to make a red build pass — fix the missing tests instead. + ### Test Guidelines - Unit tests should not require remote nodes or running game servers diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d18b50c..2a82aa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,9 @@ jobs: shell: pwsh run: $PSVersionTable | Format-List - - name: Run CI build (analyzer + tests) + - name: Run CI build (analyzer + tests + coverage gate) shell: pwsh - run: ./build.ps1 -InstallDeps + run: ./build.ps1 -InstallDeps -Coverage - name: Upload Pester results if: always() @@ -37,3 +37,11 @@ jobs: name: pester-results path: testResults.xml if-no-files-found: warn + + - name: Upload coverage report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: coverage-report + path: coverage.xml + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index 9ba842d..03698a6 100644 --- a/.gitignore +++ b/.gitignore @@ -161,6 +161,8 @@ _TeamCity* coverage*.json coverage*.xml coverage*.info +# ...but the committed ratchet baseline IS tracked (see build.ps1 Gate 3 / .github/CONTRIBUTING.md). +!coverage-baseline.json # Visual Studio code coverage results *.coverage diff --git a/build.ps1 b/build.ps1 index 4dc31a3..cab19ab 100644 --- a/build.ps1 +++ b/build.ps1 @@ -25,17 +25,27 @@ .PARAMETER ExcludeTag Pester tags to exclude (default: E2E, Integration -- the infra-dependent suites). +.PARAMETER Coverage + Measure JaCoCo code coverage over Modules/ during the (same) Pester run and enforce the + ratchet gate against coverage-baseline.json: the build FAILS if overall coverage drops more + than a small tolerance below the committed baseline, so coverage can only hold or climb. CI + passes this; local quick runs can omit it (coverage instrumentation adds wall-clock, and the + gate only applies when coverage was actually measured). Writes coverage.xml (gitignored). + +.EXAMPLE + ./build.ps1 -InstallDeps -Coverage # what CI runs .EXAMPLE - ./build.ps1 -InstallDeps # what CI runs + ./build.ps1 # local quick run (deps present, no coverage gate) .EXAMPLE - ./build.ps1 # local, deps already present + ./build.ps1 -Coverage # local run WITH the coverage ratchet gate .OUTPUTS - Exit code 0 on success, 1 on any analyzer-error/test failure. + Exit code 0 on success, 1 on any analyzer-error/test/coverage-gate failure. #> [CmdletBinding()] param( [switch]$InstallDeps, - [string[]]$ExcludeTag = @('E2E', 'Integration') + [string[]]$ExcludeTag = @('E2E', 'Integration'), + [switch]$Coverage ) $ErrorActionPreference = 'Stop' @@ -134,6 +144,19 @@ $conf.TestResult.Enabled = $true $conf.TestResult.OutputFormat = 'NUnitXml' $conf.TestResult.OutputPath = Join-Path $RepoRoot 'testResults.xml' +# Coverage is folded into THIS Pester run (one suite execution, not two) so CI time stays +# reasonable. It is opt-in via -Coverage because breakpoint instrumentation adds measurable +# wall-clock; the ratchet gate below only runs when coverage was actually measured. +$coverageXml = Join-Path $RepoRoot 'coverage.xml' +if ($Coverage) { + $conf.CodeCoverage.Enabled = $true + $conf.CodeCoverage.Path = Join-Path $RepoRoot 'Modules' # production code only, not Tests/ + $conf.CodeCoverage.OutputFormat = 'JaCoCo' # build-server-friendly report + $conf.CodeCoverage.OutputPath = $coverageXml # gitignored; uploaded as a CI artifact + # RecursePaths/SingleHitBreakpoints/ExcludeTests all default true (verified on Pester 5.7.1): + # recurse the module tree, drop each breakpoint once hit (faster), and don't count test files. +} + $result = Invoke-Pester -Configuration $conf $ran = $result.PassedCount + $result.FailedCount if ($result.Result -ne 'Passed') { @@ -155,6 +178,98 @@ else { Write-Host "Pester OK: $($result.PassedCount) passed, $($result.SkippedCount) skipped." } +# ── Gate 3: Coverage ratchet ────────────────────────────────────────────────── +# Only runs when coverage was measured (-Coverage). Compares the run's overall % against the +# committed baseline in coverage-baseline.json and FAILS the build if it dropped more than a +# tolerance below baseline. The tolerance absorbs measurement jitter: a few lines hit by timing- +# or environment-dependent paths flip between runs, nudging the % by fractions of a point. The +# baseline only moves UP, and only by a deliberate human commit (see the "ratchet" hint below). +if ($Coverage) { + Write-Host '' + Write-Host '== Coverage ratchet ==' + + # Tolerance (percentage points) below baseline that is still accepted. Small enough that a + # real regression (a whole function left untested) trips it, large enough to ride out jitter. + $coverageTolerance = 0.75 + + # How far above baseline before we suggest raising it. Avoids nagging on every fractional gain. + $ratchetHintMargin = 1.0 + + $cc = $result.CodeCoverage + if (-not $cc) { + # CodeCoverage was requested but the run produced no coverage object -- treat as a hard + # failure rather than a silent pass (a green build that measured nothing is the trap). + Write-Host 'FAIL: -Coverage was set but no coverage data was produced.' + $failed = $true + } + else { + $currentOverall = [math]::Round($cc.CoveragePercent, 2) + + # Per-module % from command-level hit/miss data, grouped by the module folder name (the + # path segment right after Modules/). Lets us NAME which modules regressed on a drop. + $cmds = @( + @($cc.CommandsExecuted | ForEach-Object { [pscustomobject]@{ File = $_.File; Hit = $true } }) + + @($cc.CommandsMissed | ForEach-Object { [pscustomobject]@{ File = $_.File; Hit = $false } }) + ) + $currentPerModule = [ordered]@{} + $cmds | Group-Object { + if ($_.File -match '[\\/]Modules[\\/]([^\\/]+)[\\/]') { $Matches[1] } else { 'OTHER' } + } | Sort-Object Name | ForEach-Object { + $total = $_.Count + $hit = @($_.Group | Where-Object Hit).Count + $currentPerModule[$_.Name] = if ($total) { [math]::Round(100.0 * $hit / $total, 2) } else { 0.0 } + } + + # Load the committed ratchet baseline. + $baselinePath = Join-Path $RepoRoot 'coverage-baseline.json' + if (-not (Test-Path -LiteralPath $baselinePath)) { + Write-Host "FAIL: coverage baseline not found at $baselinePath (run with -Coverage and commit the file)." + $failed = $true + } + else { + $baseline = Get-Content -LiteralPath $baselinePath -Raw | ConvertFrom-Json + $baseOverall = [double]$baseline.overall + $floor = [math]::Round($baseOverall - $coverageTolerance, 2) + + Write-Host ("Coverage: {0}% overall (baseline {1}%, floor {2}% with {3}pt tolerance)." -f ` + $currentOverall, $baseOverall, $floor, $coverageTolerance) + + if ($currentOverall -lt $floor) { + # DROP below the allowed floor -> fail and name the modules that regressed so the + # author knows where to add tests (or which module's deletion shifted the ratio). + Write-Host ("FAIL: coverage {0}% is below baseline {1}% (floor {2}%)." -f ` + $currentOverall, $baseOverall, $floor) + $regressed = @() + if ($baseline.perModule) { + foreach ($name in $currentPerModule.Keys) { + $bm = $baseline.perModule.$name + if ($null -ne $bm -and $currentPerModule[$name] -lt ([double]$bm - $coverageTolerance)) { + $regressed += '{0} {1}% -> {2}%' -f $name, ([double]$bm), $currentPerModule[$name] + } + } + } + if ($regressed.Count) { + Write-Host ' regressed modules:' + $regressed | ForEach-Object { Write-Host " $_" } + } + else { + Write-Host ' (no single module crossed the per-module tolerance; the overall ratio shifted -- e.g. new uncovered code or a covered module removed).' + } + $failed = $true + } + else { + Write-Host 'Coverage OK vs baseline.' + if ($currentOverall -ge ($baseOverall + $ratchetHintMargin)) { + # INCREASE beyond the margin: nudge -- but do NOT auto-raise. Raising the + # baseline is a deliberate commit so the new floor is reviewed and intentional. + Write-Host (" ratchet: baseline can be raised to {0}% (currently {1}%). Update coverage-baseline.json in a commit to lock it in." -f ` + $currentOverall, $baseOverall) + } + } + } + } +} + # ── Result ─────────────────────────────────────────────────────────────────── Write-Host '' if ($failed) { diff --git a/coverage-baseline.json b/coverage-baseline.json new file mode 100644 index 0000000..b1e5bfe --- /dev/null +++ b/coverage-baseline.json @@ -0,0 +1,22 @@ +{ + "_comment": "Code-coverage ratchet baseline for build.ps1 Gate 3. 'overall' is Pester's JaCoCo CoveragePercent over Modules/; 'perModule' is per-module command hit-ratio (same computation the gate uses at runtime, used only to name regressed modules). Raise these numbers in a deliberate commit when build.ps1 prints the 'ratchet: baseline can be raised' hint. See .github/CONTRIBUTING.md. Measured on 1199 passing tests.", + "overall": 57.31, + "perModule": { + "BackupEngine": 49.5, + "CompressionManager": 39.54, + "ConfigManager": 66.37, + "CredentialManager": 72.0, + "IntegrityManager": 79.44, + "LoadMonitor": 57.52, + "Logger": 59.6, + "ManifestManager": 80.28, + "MetricsCollector": 73.03, + "NetworkThrottle": 77.25, + "NotificationManager": 89.39, + "RemoteManager": 45.65, + "RestoreEngine": 39.76, + "SchedulerManager": 87.5, + "VRageAPI": 46.4, + "VSSManager": 13.73 + } +} From 67eb24125fa219536f74ec1e12d6db52e9564a84 Mon Sep 17 00:00:00 2001 From: Steven Cady Date: Thu, 25 Jun 2026 19:10:54 -0500 Subject: [PATCH 2/2] ci(#17): enforce per-module coverage floors + absolute floor; tighten tolerance; -CodeCoverage alias; doc fixes (8-lens) Co-Authored-By: Claude Opus 4.8 --- .github/CONTRIBUTING.md | 53 ++++++++---- build.ps1 | 181 +++++++++++++++++++++++++++++++--------- 2 files changed, 180 insertions(+), 54 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 795dce4..7254b88 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -166,29 +166,52 @@ Invoke-Pester -Path ./Tests/ConfigManager.Tests.ps1 -Output Detailed ### Code coverage gate and ratchet -CI runs `build.ps1 -Coverage`, which folds JaCoCo code coverage over `Modules/` into the **same** -Pester run (no second pass) and writes `coverage.xml` (gitignored; uploaded as the `coverage-report` -CI artifact next to `pester-results`/`testResults.xml`). Coverage is **opt-in** via `-Coverage` -because the breakpoint instrumentation adds roughly 30% wall-clock to the suite; a plain -`./build.ps1` quick run skips it, and the gate only applies when coverage was actually measured. +CI runs `build.ps1 -Coverage` (alias `-CodeCoverage`), which folds JaCoCo code coverage over +`Modules/` into the **same** Pester run (no second pass) and writes `coverage.xml` (gitignored; +uploaded as the `coverage-report` CI artifact next to `pester-results`/`testResults.xml`). Coverage +is **opt-in** because the breakpoint instrumentation adds measurable wall-clock — about **+14s** +on the current suite (~44s → ~58s, roughly a 30% increase) — so a plain `./build.ps1` quick run +skips it, and the gate only applies when coverage was actually measured. -The gate is a **ratchet**: coverage can hold or climb, never silently fall. +The gate is a **ratchet**: coverage can hold or climb, never silently fall. It enforces **three** +independent checks so no single module can quietly shed its tests behind another module's gains, +and so the baseline itself can't be edited down to disarm the gate: - The committed baseline lives in **`coverage-baseline.json`** (`{ "overall": , "perModule": { ... } }`). -- The build **FAILS** if the measured overall coverage drops more than the tolerance (0.75 pts, to - absorb run-to-run jitter from timing-dependent line hits) below `overall`. On a drop it prints - `coverage X% is below baseline Y%` and names the modules that regressed. + `overall` is Pester's JaCoCo overall %; each `perModule` value is that module's command hit-ratio + (the same number the gate computes at runtime). +- **Overall ratchet** — the build **FAILS** if measured overall drops more than the tolerance + (**0.30 pts** — ~15× the ~0.02pt run-to-run jitter) below `overall`. +- **Per-module ratchet** — the build also **FAILS** if **any** module's hit-ratio drops more than + the same 0.30 pt tolerance below its **own** `perModule` baseline. This runs **every** build (not + only when overall falls), so a module losing its tests trips the gate even if overall stays flat. + A baseline module that produces **no** coverage (deleted, or its tests stopped running) also fails; + a **new** module not yet in the map prints a `WARN` telling you to add its entry. +- **Absolute hard floor** — independent of the baseline, the build **FAILS** if the measured overall + (or the committed `overall` itself) is below **50%** (a documented constant well under the real + ~57.31%). This catches a gutted suite *and* a baseline tampered down to a tiny number. - When coverage rises more than ~1 pt above the baseline, the build prints a `ratchet: baseline can be raised to X%` hint. It does **not** auto-raise — see below. **Raising the baseline (do this when you add tests that lift coverage):** -1. Run `./build.ps1 -Coverage` and note the printed overall % (and the ratchet hint). -2. Edit `coverage-baseline.json`: set `overall` to the new number, and update the relevant - `perModule` entries (the per-module values are the module command hit-ratios the gate prints). -3. Commit it in the same PR. Raising the floor is deliberate so the new minimum is reviewed. - -Never *lower* the baseline to make a red build pass — fix the missing tests instead. +1. Run `./build.ps1 -Coverage`. Note the printed overall % (and the ratchet hint). The build also + prints each baseline module's current hit-ratio in the `== Coverage ratchet ==` block, plus a + `WARN` line listing any **new** module not yet in the map. +2. Edit `coverage-baseline.json`: + - Set `overall` to the new printed overall %. + - **Update the `perModule` entries too** — the ratchet hint only mentions the overall, but the + per-module floors are enforced independently, so raise each module you improved to its new + current value, and **add an entry for any module the build WARNed was missing** (use the value + it printed). Leaving a `perModule` value stale just keeps an old (lower) floor for that module; + omitting a new module entirely means it stays ungated at the per-module level until you add it. + - If a module was intentionally **removed** from `Modules/`, delete its `perModule` entry (the + build FAILs on a baseline module that produced no coverage). +3. Commit it in the same PR. Raising the floor — overall *and* per-module — is deliberate so the new + minimum is reviewed. + +Never *lower* the `overall` or a `perModule` value to make a red build pass — fix the missing tests +instead. (A baseline `overall` below the 50% absolute floor is rejected outright as tampered.) ### Test Guidelines diff --git a/build.ps1 b/build.ps1 index cab19ab..67b1907 100644 --- a/build.ps1 +++ b/build.ps1 @@ -27,10 +27,17 @@ .PARAMETER Coverage Measure JaCoCo code coverage over Modules/ during the (same) Pester run and enforce the - ratchet gate against coverage-baseline.json: the build FAILS if overall coverage drops more - than a small tolerance below the committed baseline, so coverage can only hold or climb. CI - passes this; local quick runs can omit it (coverage instrumentation adds wall-clock, and the - gate only applies when coverage was actually measured). Writes coverage.xml (gitignored). + ratchet gate against coverage-baseline.json. The gate FAILS the build if ANY of these trip, + so coverage can only hold or climb and no single module can quietly lose its tests: + * OVERALL drops more than the tolerance below the committed baseline overall; + * ANY PER-MODULE hit-ratio drops more than the tolerance below its own committed baseline + (a module losing its tests fails even if the overall ratio stays within tolerance); + * the measured overall (or the committed baseline overall) falls below an ABSOLUTE hard + floor that does not depend on the baseline -- so editing the baseline down to a tiny + number can't silently disarm the gate. + Alias: -CodeCoverage. CI passes this; local quick runs can omit it (coverage instrumentation + adds wall-clock, and the gate only applies when coverage was actually measured). Writes + coverage.xml (gitignored). .EXAMPLE ./build.ps1 -InstallDeps -Coverage # what CI runs @@ -45,6 +52,8 @@ param( [switch]$InstallDeps, [string[]]$ExcludeTag = @('E2E', 'Integration'), + # Alias 'CodeCoverage' so the issue-#17 wording (-CodeCoverage) works too. Same switch. + [Alias('CodeCoverage')] [switch]$Coverage ) @@ -179,18 +188,37 @@ else { } # ── Gate 3: Coverage ratchet ────────────────────────────────────────────────── -# Only runs when coverage was measured (-Coverage). Compares the run's overall % against the -# committed baseline in coverage-baseline.json and FAILS the build if it dropped more than a -# tolerance below baseline. The tolerance absorbs measurement jitter: a few lines hit by timing- -# or environment-dependent paths flip between runs, nudging the % by fractions of a point. The -# baseline only moves UP, and only by a deliberate human commit (see the "ratchet" hint below). +# Only runs when coverage was measured (-Coverage). The gate is deliberately un-gameable: it +# enforces THREE independent things against coverage-baseline.json so coverage can only hold or +# climb and no single module can quietly shed its tests behind another's gains: +# 1. OVERALL ratchet: measured overall must stay within $coverageTolerance of baseline.overall. +# 2. PER-MODULE ratchet: EVERY module in baseline.perModule must stay within $coverageTolerance +# of its OWN baseline -- enforced every run, not merely reported on an overall failure. A +# module losing its tests fails the build even when the overall ratio rides within tolerance +# (one module's gain can no longer mask another's collapse). +# 3. ABSOLUTE hard floor ($coverageAbsoluteFloor): a baseline-independent minimum applied to BOTH +# the measured overall AND the committed baseline.overall. This catches a gutted suite and a +# tampered baseline (e.g. someone editing overall down to a tiny number to disarm the gate) -- +# the floor is well below the real ~57% so it never false-trips, but a baseline below it is +# rejected as implausible rather than trusted. +# The tolerance absorbs measurement jitter: a few lines hit by timing-/environment-dependent paths +# flip between runs, nudging the % by fractions of a point. The baseline only moves UP, and only by +# a deliberate human commit (see the "ratchet" hint below and .github/CONTRIBUTING.md). if ($Coverage) { Write-Host '' Write-Host '== Coverage ratchet ==' - # Tolerance (percentage points) below baseline that is still accepted. Small enough that a - # real regression (a whole function left untested) trips it, large enough to ride out jitter. - $coverageTolerance = 0.75 + # Tolerance (percentage points) below baseline still accepted, for BOTH the overall and each + # per-module floor. Measured run-to-run jitter is ~0.02pt, so 0.3pt is ~15x the noise -- wide + # enough to ride out that jitter, tight enough that real erosion (a function/test removed) trips + # it promptly. Named constant: change it here, in one place, for both gates. + $coverageTolerance = 0.30 + + # Absolute, BASELINE-INDEPENDENT hard floor (percentage points). Documented constant set well + # below the real measured overall (~57.31%) so it never false-trips, yet high enough to catch a + # gutted suite or a baseline that was edited down to defeat the ratchet. Applied to the measured + # overall AND to baseline.overall (an implausibly low baseline is rejected, not trusted). + $coverageAbsoluteFloor = 50.0 # How far above baseline before we suggest raising it. Avoids nagging on every fractional gain. $ratchetHintMargin = 1.0 @@ -206,7 +234,8 @@ if ($Coverage) { $currentOverall = [math]::Round($cc.CoveragePercent, 2) # Per-module % from command-level hit/miss data, grouped by the module folder name (the - # path segment right after Modules/). Lets us NAME which modules regressed on a drop. + # path segment right after Modules/). This is BOTH the per-module gate input and how we + # name regressors. Stored as a hashtable for O(1) presence checks against the baseline. $cmds = @( @($cc.CommandsExecuted | ForEach-Object { [pscustomobject]@{ File = $_.File; Hit = $true } }) + @($cc.CommandsMissed | ForEach-Object { [pscustomobject]@{ File = $_.File; Hit = $false } }) @@ -231,38 +260,112 @@ if ($Coverage) { $baseOverall = [double]$baseline.overall $floor = [math]::Round($baseOverall - $coverageTolerance, 2) - Write-Host ("Coverage: {0}% overall (baseline {1}%, floor {2}% with {3}pt tolerance)." -f ` - $currentOverall, $baseOverall, $floor, $coverageTolerance) - - if ($currentOverall -lt $floor) { - # DROP below the allowed floor -> fail and name the modules that regressed so the - # author knows where to add tests (or which module's deletion shifted the ratio). - Write-Host ("FAIL: coverage {0}% is below baseline {1}% (floor {2}%)." -f ` - $currentOverall, $baseOverall, $floor) - $regressed = @() - if ($baseline.perModule) { - foreach ($name in $currentPerModule.Keys) { - $bm = $baseline.perModule.$name - if ($null -ne $bm -and $currentPerModule[$name] -lt ([double]$bm - $coverageTolerance)) { - $regressed += '{0} {1}% -> {2}%' -f $name, ([double]$bm), $currentPerModule[$name] - } + Write-Host ("Coverage: {0}% overall (baseline {1}%, floor {2}% with {3}pt tolerance; absolute floor {4}%)." -f ` + $currentOverall, $baseOverall, $floor, $coverageTolerance, $coverageAbsoluteFloor) + + # ---- Baseline sanity (tamper guard) ------------------------------------------------- + # A baseline overall below the absolute floor cannot be real for this suite (~57%); it + # is almost certainly a baseline edited down to disarm the ratchet. Reject it outright + # so a gutted baseline fails LOUDLY instead of silently lowering the floor to nothing. + if ($baseOverall -lt $coverageAbsoluteFloor) { + Write-Host ("FAIL: baseline overall {0}% is below the absolute floor {1}% -- implausible for this suite; the baseline looks gutted or tampered. Restore coverage-baseline.json." -f ` + $baseOverall, $coverageAbsoluteFloor) + $failed = $true + } + + # ---- Absolute floor on the MEASURED overall (baseline-independent) ------------------- + # Independent of the baseline so that even a tampered/lowered baseline cannot let a + # genuinely collapsed suite pass. The real overall sits ~7pt above this. + if ($currentOverall -lt $coverageAbsoluteFloor) { + Write-Host ("FAIL: coverage {0}% is below the ABSOLUTE floor {1}% (baseline-independent hard minimum). The suite appears gutted." -f ` + $currentOverall, $coverageAbsoluteFloor) + $failed = $true + } + + # ---- Overall ratchet (vs committed baseline) ---------------------------------------- + $overallRegressed = $currentOverall -lt $floor + if ($overallRegressed) { + Write-Host ("FAIL: overall coverage {0}% is below baseline {1}% (floor {2}%, {3}pt tolerance)." -f ` + $currentOverall, $baseOverall, $floor, $coverageTolerance) + $failed = $true + } + + # ---- Per-module ratchet (ALWAYS enforced, not just on an overall drop) --------------- + # For EVERY module recorded in the baseline, fail if its current hit-ratio fell more + # than the tolerance below its own baseline. Running this unconditionally is the whole + # point: it catches a single module shedding tests even while the overall holds. + $perModuleRegressed = @() + $perModuleOk = @() + $newModules = @() # present on disk now but absent from the baseline + $missingModules = @() # in the baseline but no longer produced any coverage + + if (-not $baseline.perModule) { + # No per-module map at all -> cannot enforce floor #2. Treat as a baseline that must + # be regenerated rather than silently skipping half the gate. + Write-Host 'FAIL: coverage baseline has no perModule map; cannot enforce per-module floors. Regenerate coverage-baseline.json (see .github/CONTRIBUTING.md).' + $failed = $true + } + else { + # Baseline-side member names (ConvertFrom-Json gives a PSCustomObject; enumerate its + # NoteProperty names) so we can detect baseline entries with no current coverage. + $baselineModuleNames = @($baseline.perModule.PSObject.Properties.Name) + + # Enforce each baseline module's own floor; guard against a baseline module that no + # longer exists on disk (produced no commands this run) instead of comparing to nothing. + foreach ($name in $baselineModuleNames) { + $bm = [double]$baseline.perModule.$name + if (-not $currentPerModule.Contains($name)) { + $missingModules += '{0} (baseline {1}%)' -f $name, $bm + continue + } + $cur = [double]$currentPerModule[$name] + $mFloor = [math]::Round($bm - $coverageTolerance, 2) + if ($cur -lt $mFloor) { + $perModuleRegressed += '{0} {1}% -> {2}% (floor {3}%)' -f $name, $bm, $cur, $mFloor + } + else { + $perModuleOk += '{0} {1}%' -f $name, $cur } } - if ($regressed.Count) { - Write-Host ' regressed modules:' - $regressed | ForEach-Object { Write-Host " $_" } + + # A module present now but ABSENT from the baseline is a new module that needs its + # own baseline entry -- don't null-deref it, and don't let it sneak in ungated. Warn + # loudly so its floor gets committed; the overall+absolute gates still cover the run. + foreach ($name in $currentPerModule.Keys) { + if ($name -notin $baselineModuleNames) { + $newModules += '{0} {1}%' -f $name, ([double]$currentPerModule[$name]) + } } - else { - Write-Host ' (no single module crossed the per-module tolerance; the overall ratio shifted -- e.g. new uncovered code or a covered module removed).' + + if ($perModuleRegressed.Count) { + Write-Host ' FAIL: per-module coverage regressed below baseline:' + $perModuleRegressed | ForEach-Object { Write-Host " $_" } + $failed = $true + } + if ($missingModules.Count) { + # A baseline module with no coverage this run: either it was deleted (update the + # baseline) or its tests stopped running (a real regression). Fail so it's noticed. + Write-Host ' FAIL: baseline modules produced no coverage this run (deleted, or their tests stopped running):' + $missingModules | ForEach-Object { Write-Host " $_" } + Write-Host ' If a module was intentionally removed, drop its entry from coverage-baseline.json perModule.' + $failed = $true + } + if ($newModules.Count) { + Write-Host ' WARN: new module(s) not yet in the baseline perModule map -- add an entry to lock in their floor:' + $newModules | ForEach-Object { Write-Host " $_" } } - $failed = $true } - else { - Write-Host 'Coverage OK vs baseline.' + + # ---- Summary / ratchet hint --------------------------------------------------------- + if (-not $overallRegressed -and -not $perModuleRegressed.Count -and ` + -not $missingModules.Count -and $currentOverall -ge $coverageAbsoluteFloor -and ` + $baseOverall -ge $coverageAbsoluteFloor) { + Write-Host ("Coverage OK vs baseline: overall {0}% (floor {1}%) and all {2} baseline module(s) within tolerance." -f ` + $currentOverall, $floor, @($perModuleOk).Count) if ($currentOverall -ge ($baseOverall + $ratchetHintMargin)) { - # INCREASE beyond the margin: nudge -- but do NOT auto-raise. Raising the - # baseline is a deliberate commit so the new floor is reviewed and intentional. - Write-Host (" ratchet: baseline can be raised to {0}% (currently {1}%). Update coverage-baseline.json in a commit to lock it in." -f ` + # INCREASE beyond the margin: nudge -- but do NOT auto-raise. Raising the baseline + # is a deliberate commit so the new floor (overall AND perModule) is reviewed. + Write-Host (" ratchet: baseline can be raised to {0}% (currently {1}%). Update coverage-baseline.json -- BOTH overall and the relevant perModule entries -- in a commit to lock it in (see .github/CONTRIBUTING.md)." -f ` $currentOverall, $baseOverall) } }