diff --git a/.github/workflows/benchmark-history.yml b/.github/workflows/benchmark-history.yml index 4c90f7d..deb6195 100644 --- a/.github/workflows/benchmark-history.yml +++ b/.github/workflows/benchmark-history.yml @@ -5,8 +5,15 @@ name: Benchmark History # which exists to get a full ad-hoc run on demand and publishes nothing. # # Two ways in: -# * a published release, which measures that version and adds one point; -# * a manual dispatch listing refs, which measures each of them in ONE job and backfills. +# * a published release, which measures that version from its own tag and adds one point; +# * a manual dispatch listing versions, which measures each published package in ONE job. +# +# A backfill measures packages rather than tags. Both are defensible until a benchmark is added: a +# tag predating one cannot run it, so a new measurement could only ever start from the next release, +# and AbstractionCostBenchmarks would have had no history at all. Measuring the published packages +# with today's sources gives it the same history as everything else, and times every version by +# identical code besides. ktsu.SignificantNumber and ktsu.Semantics do the same, so the three +# charts are built the same way. # # The backfill running as a single job is the point rather than an optimisation. Separate runs land # on different CI hosts, and the difference between an x86-64-v3 and a v4 runner is larger than @@ -18,15 +25,10 @@ on: types: [published] workflow_dispatch: inputs: - refs: - description: "Space-separated refs to backfill, oldest first (tags, branches, or SHAs)" - required: false - default: "cd9a8227793bbd6ea791be7f6c0579ae1242eec5 v2.0.0 v2.0.1 v2.0.2 v2.0.3 v2.0.4" - type: string - labels: - description: "Optional space-separated version labels matching refs, when a ref is not a version" + versions: + description: "Space-separated released versions to backfill, oldest first" required: false - default: "1.9.0 2.0.0 2.0.1 2.0.2 2.0.3 2.0.4" + default: "1.8.0 1.9.0 2.0.0 2.0.1 2.0.2 2.0.3 2.0.4 2.0.5" type: string permissions: @@ -52,6 +54,7 @@ env: *ConstructionBenchmarks.Sanitizing *TextBenchmarks.Parse *ConversionBenchmarks.ToDouble + *AbstractionCostBenchmarks.* # Short runs: three iterations is enough for a trend line, and a release should not tie up a # runner for half an hour. benchmarks.yml is still there for a full-length run. BENCHMARK_JOB: short @@ -121,51 +124,55 @@ jobs: git worktree remove --force "$work" - - name: Measure each backfill ref + - name: Measure each backfill version if: github.event_name == 'workflow_dispatch' shell: bash env: - REFS: ${{ inputs.refs }} - LABELS: ${{ inputs.labels }} + VERSIONS: ${{ inputs.versions }} BASELINE_NS: ${{ steps.baseline.outputs.ns }} run: | set -euo pipefail - read -ra refs <<< "$REFS" - read -ra labels <<< "$LABELS" - - for index in "${!refs[@]}"; do - ref="${refs[$index]}" - label="${labels[$index]:-${ref#v}}" - work="${RUNNER_TEMP}/bench-$label" - - echo "::group::$label ($ref)" - rm -rf "$work" - git worktree add --detach "$work" "$ref" - - if [ ! -f "$work/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj" ]; then - echo "::warning::$ref has no benchmark project; skipping" - git worktree remove --force "$work" + read -ra versions <<< "$VERSIONS" + + for version in "${versions[@]}"; do + echo "::group::$version" + # Through the environment rather than a -p: switch, because BenchmarkDotNet generates + # and builds a project of its own per run, which a property passed on the command line + # does not reach. MSBuild reads environment variables as properties in every project. + # + # A version whose API the current benchmarks cannot express is reported and skipped, + # rather than failing the whole backfill after the ones before it have been measured. + if ! BenchmarkAgainstVersion="$version" dotnet run -c Release --project PreciseNumber.Benchmarks -- \ + --filter $HEADLINE_FILTER \ + --job "$BENCHMARK_JOB" \ + --artifacts "$GITHUB_WORKSPACE/$RUNS/$version"; then + echo "::warning::$version could not be benchmarked by the current suite; skipping" echo "::endgroup::" continue fi - # Each ref is measured by its own benchmark sources. Between 2.0.0 and now those - # sources are unchanged, so this compares library versions rather than harnesses. - (cd "$work" && dotnet run -c Release --project PreciseNumber.Benchmarks -- \ - --filter $HEADLINE_FILTER \ - --job "$BENCHMARK_JOB" \ - --artifacts "$GITHUB_WORKSPACE/$RUNS/$label") - - dotnet run scripts/benchmark-history.cs -- ingest \ - --history "$HISTORY" \ - --results "$RUNS/$label" \ - --version "$label" \ - --commit "$(git rev-parse --short "$ref^{commit}")" \ - --date "$(git log -1 --format=%cs "$ref")" \ - --run-id "${{ github.run_id }}" \ - --baseline-ns "$BASELINE_NS" - - git worktree remove --force "$work" + tag="v$version" + commit="" + date="" + if git rev-parse -q --verify "$tag^{commit}" >/dev/null; then + commit="$(git rev-parse --short "$tag^{commit}")" + date="$(git log -1 --format=%cs "$tag")" + fi + + # Skipped here too, and for the same reason: a package can build against these + # benchmarks and still throw from every one of them at run time, which BenchmarkDotNet + # reports as a table of NA rather than as a failure. Ingest refuses such a run, and + # the backfill carries on to the next version. + if ! dotnet run scripts/benchmark-history.cs -- ingest \ + --history "$HISTORY" \ + --results "$RUNS/$version" \ + --version "$version" \ + --commit "$commit" \ + --date "$date" \ + --run-id "${{ github.run_id }}" \ + --baseline-ns "$BASELINE_NS"; then + echo "::warning::$version produced no usable measurement; skipping" + fi echo "::endgroup::" done diff --git a/Directory.Packages.props b/Directory.Packages.props index be4d333..dbc7dfc 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,5 +6,10 @@ + + diff --git a/PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs b/PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs new file mode 100644 index 0000000..1d3ecc9 --- /dev/null +++ b/PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs @@ -0,0 +1,160 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.PreciseNumber.Benchmarks; + +using System.Globalization; + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; + +/// +/// Measures what this type costs against the same arithmetic on a bare . +/// +/// +/// +/// Every other class here answers "how long does this operation take", which is only readable next +/// to something. This one supplies the something: the primitive a caller would otherwise have +/// used. The same class, with the same loops and the same methodology, is in ktsu.SignificantNumber +/// and ktsu.Semantics, so the three answers are comparable with each other as well as with +/// . +/// +/// +/// The bare method is the BenchmarkDotNet baseline, so the answer is the Ratio column rather +/// than two rows divided by hand. Unlike the wrapper types built on this one, a ratio here is not +/// expected to be 1.00 and is not a defect when it is not: arbitrary precision is a cost paid for +/// something a cannot do at all. What the number is for is watching that cost +/// across releases. +/// +/// +/// Why these are loops. A single operation over operands that do not change is +/// loop-invariant, and the JIT hoists it out of the measurement entirely — for +/// that leaves a method indistinguishable from an empty one, and a ratio +/// against an empty method means nothing. Here each iteration feeds the next, so there is nothing +/// to hoist and both sides are measurable. +/// +/// +/// Which way the loop biases the answer. Both sides pay the same counter and branch, and it +/// is a dependency chain, so most of that overlaps the arithmetic; whatever does not is added +/// equally to numerator and denominator and pulls the ratio toward 1.00. A ratio here is therefore +/// a floor on the real cost rather than the whole of it. +/// +/// +/// Why the operands stay bounded, and why they are short. This type carries as many digits +/// as the arithmetic produces, so a chain that compounded its operand would measure that growth +/// rather than the operation; both loops accumulate instead. The operands are also chosen to be +/// values a can hold, so the two sides are doing the same arithmetic on the +/// same numbers rather than being handed different problems. How the cost grows with digits is a +/// different question, and answers it across its +/// Digits axis. +/// +/// +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class AbstractionCostBenchmarks +{ + /// + /// Operations per invocation. Enough that the loop's own cost is a small share of the work, + /// few enough that the arbitrary-precision side still finishes an iteration promptly. + /// + private const int Operations = 256; + + private const string SeedText = "1234.5678901234"; + private const string StepText = "0.0009765625"; + private const string OtherText = "3.14159265358979"; + + private double bareSeed; + private double bareStep; + private double bareOther; + + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. The backfill measures + // those releases too, so the file has to compile against both shapes. + private PreciseNumber preciseSeed = default!; + private PreciseNumber preciseStep = default!; + private PreciseNumber preciseOther = default!; + + /// + /// Prepares the operands, parsed from the same text on both sides. + /// + [GlobalSetup] + public void Setup() + { + bareSeed = double.Parse(SeedText, CultureInfo.InvariantCulture); + bareStep = double.Parse(StepText, CultureInfo.InvariantCulture); + bareOther = double.Parse(OtherText, CultureInfo.InvariantCulture); + + preciseSeed = PreciseNumber.Parse(SeedText, CultureInfo.InvariantCulture); + preciseStep = PreciseNumber.Parse(StepText, CultureInfo.InvariantCulture); + preciseOther = PreciseNumber.Parse(OtherText, CultureInfo.InvariantCulture); + } + + /// Adds along a chain, on a bare double. + /// The accumulated value. + [BenchmarkCategory("Add")] + [Benchmark(Baseline = true, OperationsPerInvoke = Operations)] + public double BareAdd() + { + double accumulator = bareSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += bareStep; + } + + return accumulator; + } + + /// Adds along the same chain, on this type. + /// The accumulated value. + [BenchmarkCategory("Add")] + [Benchmark(OperationsPerInvoke = Operations)] + public PreciseNumber PreciseAdd() + { + PreciseNumber accumulator = preciseSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += preciseStep; + } + + return accumulator; + } + + /// Multiplies and accumulates, on a bare double. + /// The accumulated value. + [BenchmarkCategory("Multiply")] + [Benchmark(Baseline = true, OperationsPerInvoke = Operations)] + public double BareMultiply() + { + double accumulator = 0d; + double value = bareSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += value * bareOther; + value += bareStep; + } + + return accumulator; + } + + /// Multiplies and accumulates over the same values, on this type. + /// The accumulated value. + [BenchmarkCategory("Multiply")] + [Benchmark(OperationsPerInvoke = Operations)] + public PreciseNumber PreciseMultiply() + { + PreciseNumber accumulator = PreciseNumber.Zero; + PreciseNumber value = preciseSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += value * preciseOther; + value += preciseStep; + } + + return accumulator; + } +} diff --git a/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj b/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj index c6aa6ec..fca1d25 100644 --- a/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj +++ b/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj @@ -29,8 +29,34 @@ - + + + + + + + + + + diff --git a/PreciseNumber.Benchmarks/README.md b/PreciseNumber.Benchmarks/README.md index f9dd876..62e42ec 100644 --- a/PreciseNumber.Benchmarks/README.md +++ b/PreciseNumber.Benchmarks/README.md @@ -50,6 +50,39 @@ each digit separately looks fine at 8 digits and falls apart at 200. Reading a t apart from operands in the same decade, because aligning two exponents is its own cost, distinct from the size of the operands. +## What this type costs against a bare double + +`AbstractionCostBenchmarks` is the one benchmark here whose answer is a ratio rather than a +duration. Every other class says how long an operation takes, which is only readable beside +something; this supplies the something — the primitive a caller would otherwise have used. + +The same class, with the same loops and the same methodology, is in `ktsu.SignificantNumber` and +`ktsu.Semantics`, so the three libraries answer one question the same way and their answers are +comparable with each other as well as with `double`. + +| release | `Add` | `Multiply` | +|---|---|---| +| 1.8.0 | 99.7× | 293.9× | +| 1.9.0 | 99.7× | 288.2× | +| 2.0.0 | 82.8× | 248.8× | +| 2.0.5 | 83.6× | 249.8× | + +Becoming a value type in 2.0 took about 15% off the price of arbitrary precision, and six releases +have held it there. **The ratio is not expected to be 1 and is not a defect for being large** — a +`double` cannot do this at all. What the chart's third section is for is noticing the day it moves. + +Three things decide how the number should be read: + +- **These are loops.** A single operation over operands that do not change is loop-invariant and + the JIT hoists it out, which would leave the `double` side indistinguishable from an empty method + and the ratio meaningless. Each iteration feeds the next, so there is nothing to hoist. +- **The loop's own cost biases toward 1**, being paid by both sides, so a ratio is a floor on the + real cost rather than the whole of it. +- **Both loops accumulate rather than compound**, because this type carries as many digits as the + arithmetic produces and a compounding chain would measure that growth instead of the operation. + How the cost grows with digits is a different question, and `ArithmeticBenchmarks` answers it + across the `Digits` axis. + ## Reading the results Allocation is reported next to time. Both matter here, and they trade against each other: every diff --git a/docs/benchmarks/history.json b/docs/benchmarks/history.json index 2d37639..6d70308 100644 --- a/docs/benchmarks/history.json +++ b/docs/benchmarks/history.json @@ -7,96 +7,106 @@ "date": "2026-09-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9865, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3522, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 98.3125, + "allocatedBytes": 72 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 397.4461, + "allocatedBytes": 243 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 123.1859, + "meanNs": 116.6638, "allocatedBytes": 104 }, "Digits=30": { - "meanNs": 164.7113, + "meanNs": 161.361, "allocatedBytes": 120 }, "Digits=200": { - "meanNs": 402.6075, + "meanNs": 420.159, "allocatedBytes": 264 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 794.4781, + "meanNs": 817.6077, "allocatedBytes": 264 }, "Digits=30": { - "meanNs": 889.3647, + "meanNs": 858.7159, "allocatedBytes": 312 }, "Digits=200": { - "meanNs": 995.529, + "meanNs": 958.3089, "allocatedBytes": 456 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 81.9846, + "meanNs": 87.7939, "allocatedBytes": 72 }, "Digits=30": { - "meanNs": 184.0505, + "meanNs": 183.6178, "allocatedBytes": 96 }, "Digits=200": { - "meanNs": 1050.0351, + "meanNs": 1057.627, "allocatedBytes": 232 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.5, + "meanNs": 4.6545, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 4.7745, + "meanNs": 4.7675, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 5.1859, + "meanNs": 4.7959, "allocatedBytes": 0 } }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 35.0957, - "allocatedBytes": 40 - }, - "Digits=30": { - "meanNs": 76.4736, - "allocatedBytes": 40 - }, - "Digits=200": { - "meanNs": 254.6939, - "allocatedBytes": 40 - } - }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 20.149, + "meanNs": 21.0712, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 199.5932, + "meanNs": 207.9911, "allocatedBytes": 40 }, "Digits=30": { - "meanNs": 480.1194, + "meanNs": 488.4566, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 2534.6838, + "meanNs": 2561.4915, "allocatedBytes": 152 } } @@ -108,96 +118,106 @@ "date": "2026-09-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9939, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.356, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 99.0794, + "allocatedBytes": 72 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 390.7421, + "allocatedBytes": 243 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 114.0296, + "meanNs": 121.8675, "allocatedBytes": 104 }, "Digits=30": { - "meanNs": 156.3233, + "meanNs": 164.1578, "allocatedBytes": 120 }, "Digits=200": { - "meanNs": 428.4019, + "meanNs": 431.5375, "allocatedBytes": 264 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 403.9122, + "meanNs": 417.5664, "allocatedBytes": 192 }, "Digits=30": { - "meanNs": 613.6075, + "meanNs": 619.2303, "allocatedBytes": 240 }, "Digits=200": { - "meanNs": 2375.4439, + "meanNs": 3484.2076, "allocatedBytes": 568 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 84.6192, + "meanNs": 86.8419, "allocatedBytes": 72 }, "Digits=30": { - "meanNs": 179.2593, + "meanNs": 185.772, "allocatedBytes": 96 }, "Digits=200": { - "meanNs": 1029.5614, + "meanNs": 1051.0365, "allocatedBytes": 232 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.55, + "meanNs": 4.6033, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 4.7625, + "meanNs": 4.6786, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 4.5483, + "meanNs": 4.7639, "allocatedBytes": 0 } }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 35.4251, - "allocatedBytes": 40 - }, - "Digits=30": { - "meanNs": 80.1594, - "allocatedBytes": 40 - }, - "Digits=200": { - "meanNs": 262.4124, - "allocatedBytes": 40 - } - }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 20.7065, + "meanNs": 20.3028, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 205.5582, + "meanNs": 206.2804, "allocatedBytes": 40 }, "Digits=30": { - "meanNs": 490.8118, + "meanNs": 490.0897, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 2503.218, + "meanNs": 2644.6797, "allocatedBytes": 152 } } @@ -209,96 +229,106 @@ "date": "2026-09-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9959, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3698, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.487, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 340.8541, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 100.4854, + "meanNs": 102.2745, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 145.0197, + "meanNs": 152.0189, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 406.8626, + "meanNs": 406.6596, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 421.971, + "meanNs": 431.6098, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 649.0157, + "meanNs": 625.1701, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2501.2133, + "meanNs": 2410.6448, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 70.0239, + "meanNs": 69.667, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 175.1806, + "meanNs": 169.1872, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1025.2062, + "meanNs": 1025.3157, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 3.1494, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 4.2118, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.589, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.0883, + "meanNs": 3.9819, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 63.0727, + "meanNs": 4.1473, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 242.1056, + "meanNs": 3.5786, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.749, + "meanNs": 5.2676, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 187.9601, + "meanNs": 191.5534, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 479.94, + "meanNs": 482.5276, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2528.7065, + "meanNs": 2563.0209, "allocatedBytes": 112 } } @@ -310,96 +340,106 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9927, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.365, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.6058, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 338.9373, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 100.8056, + "meanNs": 103.9937, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 148.6598, + "meanNs": 147.0839, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 576.7779, + "meanNs": 406.8272, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 432.8415, + "meanNs": 447.1332, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 630.3228, + "meanNs": 653.7294, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2408.9761, + "meanNs": 3460.0593, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 71.7227, + "meanNs": 70.7119, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 168.2777, + "meanNs": 168.7656, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1036.3418, + "meanNs": 1028.2038, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 3.9038, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 4.1531, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 4.2655, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.3621, + "meanNs": 3.6292, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 63.2541, + "meanNs": 4.2232, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 242.4946, + "meanNs": 3.8236, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.0037, + "meanNs": 5.6916, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 186.071, + "meanNs": 191.3243, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 486.0585, + "meanNs": 484.8135, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2492.1256, + "meanNs": 2494.1918, "allocatedBytes": 112 } } @@ -411,96 +451,106 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9882, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3598, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.8552, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 334.529, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 101.1571, + "meanNs": 103.792, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 147.8035, + "meanNs": 150.2509, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 397.9905, + "meanNs": 406.1234, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 421.5628, + "meanNs": 432.5153, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 648.9157, + "meanNs": 643.1053, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2435.5906, + "meanNs": 2408.4417, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 74.7106, + "meanNs": 85.3321, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 169.6895, + "meanNs": 172.561, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1011.6622, + "meanNs": 1021.9672, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.0939, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.8344, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.6637, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 20.3191, + "meanNs": 3.617, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 64.0998, + "meanNs": 3.8082, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 244.0778, + "meanNs": 3.5699, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.3519, + "meanNs": 5.3681, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 185.887, + "meanNs": 193.862, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 489.6898, + "meanNs": 489.4994, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2449.7325, + "meanNs": 2476.3004, "allocatedBytes": 112 } } @@ -512,96 +562,106 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9904, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3589, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.8596, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 343.61, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 106.0063, + "meanNs": 104.5388, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 145.0424, + "meanNs": 150.2235, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 400.2685, + "meanNs": 407.8852, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 423.013, + "meanNs": 423.821, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 641.3537, + "meanNs": 625.2752, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 3414.9285, + "meanNs": 3511.3953, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 67.2843, + "meanNs": 69.7463, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 168.3793, + "meanNs": 169.4646, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1019.9757, + "meanNs": 1016.2448, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.4645, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.8372, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.5919, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.6194, + "meanNs": 3.5734, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 67.0253, + "meanNs": 4.4563, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 247.2198, + "meanNs": 3.9594, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.1391, + "meanNs": 4.8, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 190.2204, + "meanNs": 187.8255, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 485.4024, + "meanNs": 481.0221, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2529.4197, + "meanNs": 2547.2377, "allocatedBytes": 112 } } @@ -613,96 +673,106 @@ "date": "2026-09-15", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9948, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3565, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.7114, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 333.9983, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 103.5012, + "meanNs": 104.7875, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 147.654, + "meanNs": 147.415, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 395.7874, + "meanNs": 408.9774, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 419.0343, + "meanNs": 434.0507, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 627.7278, + "meanNs": 637.4499, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2440.4735, + "meanNs": 3422.1123, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 69.7283, + "meanNs": 68.155, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 165.934, + "meanNs": 171.208, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1027.7703, + "meanNs": 1026.3715, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 3.5588, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.5576, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.6354, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.7243, + "meanNs": 4.1605, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 63.6485, + "meanNs": 3.5677, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 241.9847, + "meanNs": 4.2016, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.3881, + "meanNs": 5.4118, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 189.778, + "meanNs": 189.268, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 477.5123, + "meanNs": 475.3413, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2512.168, + "meanNs": 2369.1519, "allocatedBytes": 112 } } @@ -714,96 +784,106 @@ "date": "2026-09-16", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9912, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3655, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.8329, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 341.1631, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 104.0341, + "meanNs": 106.8633, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 144.1285, + "meanNs": 146.4693, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 397.1837, + "meanNs": 408.8275, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 425.7019, + "meanNs": 430.0093, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 615.669, + "meanNs": 646.074, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 3372.9288, + "meanNs": 2476.1516, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 68.7651, + "meanNs": 68.9266, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 167.7462, + "meanNs": 173.2572, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1010.8013, + "meanNs": 1021.0018, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.2163, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.8202, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.5939, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.1214, + "meanNs": 3.5867, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 62.9138, + "meanNs": 3.7923, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 241.3521, + "meanNs": 4.1539, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 4.8894, + "meanNs": 4.7214, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 194.2737, + "meanNs": 196.5275, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 476.8454, + "meanNs": 481.0981, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2442.8716, + "meanNs": 2521.0782, "allocatedBytes": 112 } } diff --git a/docs/benchmarks/performance-dark.svg b/docs/benchmarks/performance-dark.svg index 28fc8de..479cb33 100644 --- a/docs/benchmarks/performance-dark.svg +++ b/docs/benchmarks/performance-dark.svg @@ -1,4 +1,4 @@ - + - + PreciseNumber performance by release 8 releases · newest 2.0.5 · 2026-09-16 @@ -68,18 +68,7 @@ 0 B 0 B Construct (30 digits) - - - - - - - - - - -0 B -40 B +not measured Parse (30 digits) @@ -111,95 +100,113 @@ Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. Add (30 digits) - - - - - - - - - -0.317× -0.362× + + + + + + + + + +0.322× +0.355× Multiply (30 digits) - - - - - - - - - -0.369× -0.405× + + + + + + + + + +0.381× +0.404× Divide (30 digits) - + - - - - - - - -1.35× -1.96× + + + + + + + +1.42× +1.89× CompareTo (30 digits) - + - - - - - - - -0.0084× + + + + + + + +0.0083× 0.0105× Construct (30 digits) - - - - - - - - - - -0.138× -0.168× +not measured Parse (30 digits) - - + + - - + + - - - -1.05× -1.06× + + + +1.06× +1.07× ToDouble - - - - - - - - - -0.0107× -0.0443× -releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 -Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. + + + + + + + + + +0.0104× +0.0463× + +Cost over the same arithmetic on a bare double +Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put. +Add + + + + + + + + + + +83.6× +99.7× +Multiply + + + + + + + + + + +250× +294× +releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 +Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. diff --git a/docs/benchmarks/performance.svg b/docs/benchmarks/performance.svg index b417420..77bb00f 100644 --- a/docs/benchmarks/performance.svg +++ b/docs/benchmarks/performance.svg @@ -1,4 +1,4 @@ - + - + PreciseNumber performance by release 8 releases · newest 2.0.5 · 2026-09-16 @@ -68,18 +68,7 @@ 0 B 0 B Construct (30 digits) - - - - - - - - - - -0 B -40 B +not measured Parse (30 digits) @@ -111,95 +100,113 @@ Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. Add (30 digits) - - - - - - - - - -0.317× -0.362× + + + + + + + + + +0.322× +0.355× Multiply (30 digits) - - - - - - - - - -0.369× -0.405× + + + + + + + + + +0.381× +0.404× Divide (30 digits) - + - - - - - - - -1.35× -1.96× + + + + + + + +1.42× +1.89× CompareTo (30 digits) - + - - - - - - - -0.0084× + + + + + + + +0.0083× 0.0105× Construct (30 digits) - - - - - - - - - - -0.138× -0.168× +not measured Parse (30 digits) - - + + - - + + - - - -1.05× -1.06× + + + +1.06× +1.07× ToDouble - - - - - - - - - -0.0107× -0.0443× -releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 -Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. + + + + + + + + + +0.0104× +0.0463× + +Cost over the same arithmetic on a bare double +Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put. +Add + + + + + + + + + + +83.6× +99.7× +Multiply + + + + + + + + + + +250× +294× +releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 +Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. diff --git a/scripts/benchmark-history.cs b/scripts/benchmark-history.cs index 408895d..8192873 100644 --- a/scripts/benchmark-history.cs +++ b/scripts/benchmark-history.cs @@ -49,12 +49,39 @@ private static readonly (string Key, string? Parameters, string Label)[] Headlin /// private static readonly Dictionary Themes = new(StringComparer.Ordinal) { - ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834"), - ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926"), + ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834", "#2e8b57"), + ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926", "#3faa71"), }; private sealed record Theme( - string Surface, string Ink, string Muted, string Grid, string Alloc, string Time); + string Surface, string Ink, string Muted, string Grid, string Alloc, string Time, string Cost); + + /// Which of the three things a section draws. + private enum Measure + { + /// Bytes allocated per operation, as stored. + Allocation, + + /// Time, divided by the reference workload from the same job. + Time, + + /// Time, divided by the paired bare-double benchmark from the same entry. + Cost, + } + + /// + /// The paired benchmarks the cost section draws, as (measured, baseline, label). + /// + /// + /// Both halves are stored like any other benchmark; the ratio is computed here rather than + /// recorded, so an entry gathered before this section existed still draws once its run + /// includes the pair, and no history has to be rewritten to change what the section shows. + /// + private static readonly (string Key, string Baseline, string Label)[] CostHeadline = + [ + ("AbstractionCostBenchmarks.PreciseAdd", "AbstractionCostBenchmarks.BareAdd", "Add"), + ("AbstractionCostBenchmarks.PreciseMultiply", "AbstractionCostBenchmarks.BareMultiply", "Multiply"), + ]; internal static int Run(string[] args) { @@ -161,6 +188,19 @@ private static int Ingest(Dictionary options) } string version = Required(options, "version"); + JsonObject benchmarks = Benchmarks(measured); + if (benchmarks.Count == 0) + { + // Reports with every row reading NA: the harness built and ran, and each benchmark + // threw. An older package whose API these benchmarks compile against but whose + // behaviour they cannot exercise does exactly this. Recording it would put a release + // on the axis with nothing under it, which reads as a release that was measured and + // found to cost nothing. + Console.Error.WriteLine( + $"No benchmark in {resultsDirectory} produced a measurement; {version} not recorded"); + return 1; + } + JsonObject record = new() { ["version"] = version, @@ -170,7 +210,7 @@ private static int Ingest(Dictionary options) ["runtime"] = runtime, ["baselineNs"] = baseline, ["runId"] = Optional(options, "run-id"), - ["benchmarks"] = Benchmarks(measured), + ["benchmarks"] = benchmarks, }; string historyPath = Required(options, "history"); @@ -390,16 +430,17 @@ private static string Draw(JsonArray entries, Theme theme) string[] labels = [.. entries.Select(entry => entry!["version"]?.GetValue() ?? "?")]; int width = Left + (Columns * CellWidth) + 24; int rows = (Headline.Length + Columns - 1) / Columns; - int height = 72 + (((34 + (rows * CellHeight)) * 2) + 54); + int costRows = (CostHeadline.Length + Columns - 1) / Columns; + int height = 72 + ((34 + (rows * CellHeight)) * 2) + 34 + (costRows * CellHeight) + 54; StringBuilder svg = new(); Preamble(svg, theme, width, height, entries); int y = 72; - foreach (bool isTime in (bool[])[false, true]) + foreach (Measure measure in (Measure[])[Measure.Allocation, Measure.Time, Measure.Cost]) { - Section(svg, theme, entries, labels.Length, y, isTime); - y += 34 + (rows * CellHeight); + Section(svg, theme, entries, labels.Length, y, measure); + y += 34 + ((measure == Measure.Cost ? costRows : rows) * CellHeight); } Footer(svg, entries, labels, y - 4); @@ -429,20 +470,53 @@ private static void Preamble(StringBuilder svg, Theme theme, int width, int heig svg.AppendLine(CultureInfo.InvariantCulture, $"""{entries.Count} releases · newest {Escape(latest["version"]?.GetValue() ?? "?")}{suffix}"""); } - private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, bool isTime) + private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, Measure measure) { - string colour = isTime ? theme.Time : theme.Alloc; - string title = isTime - ? "Time, as a multiple of a fixed reference workload" - : "Allocated bytes per operation"; - string note = isTime - ? "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster." - : "Deterministic: the same code allocates the same bytes on any machine."; + string colour = measure switch + { + Measure.Time => theme.Time, + Measure.Cost => theme.Cost, + _ => theme.Alloc, + }; + string title = measure switch + { + Measure.Time => "Time, as a multiple of a fixed reference workload", + Measure.Cost => "Cost over the same arithmetic on a bare double", + _ => "Allocated bytes per operation", + }; + string note = measure switch + { + Measure.Time => "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster.", + Measure.Cost => "Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put.", + _ => "Deterministic: the same code allocates the same bytes on any machine.", + }; svg.AppendLine(CultureInfo.InvariantCulture, $""""""); svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(title)}"""); svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(note)}"""); + if (measure == Measure.Cost) + { + for (int position = 0; position < CostHeadline.Length; position++) + { + (string key, string baseline, string label) = CostHeadline[position]; + double?[] values = [.. entries.Select(entry => Cost(entry!, key, baseline))]; + Panel( + svg, + Left + (position % Columns * CellWidth), + y + 26 + (position / Columns * CellHeight), + label, + points, + values, + true, + colour, + theme); + } + + return; + } + + bool isTime = measure == Measure.Time; for (int position = 0; position < Headline.Length; position++) { (string key, string? parameters, string label) = Headline[position]; @@ -460,6 +534,25 @@ private static void Section(StringBuilder svg, Theme theme, JsonArray entries, i } } + /// + /// One benchmark's mean divided by the mean of the bare-double benchmark beside it. + /// + /// + /// Both were measured in the same job on the same machine, so unlike the time section this + /// needs no reference workload to be comparable across runs: the denominator is the reference. + /// + private static double? Cost(JsonNode entry, string key, string baseline) + { + double? measured = Mean(entry, key); + double? divisor = Mean(entry, baseline); + return measured is not null && divisor is > 0 ? measured / divisor : null; + } + + private static double? Mean(JsonNode entry, string key) => + entry["benchmarks"]?[key] is JsonObject cases && cases.Count > 0 + ? cases.First().Value?["meanNs"]?.GetValue() + : null; + private static double? Value(JsonNode entry, string key, string? parameters, bool isTime) { if (entry["benchmarks"]?[key] is not JsonObject cases || cases.Count == 0)