From 66be68dda9f550b6b3cfb2aa3fa2b65998e9ee30 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 11:03:37 +0000 Subject: [PATCH] Measure this type against a bare double, and chart it per release [patch] Adds AbstractionCostBenchmarks and a third chart section carrying what it measures. The same class, with the same loops and the same methodology, goes into ktsu.SignificantNumber and ktsu.Semantics, so the three libraries answer one question the same way and their answers are comparable with each other as well as with double. Every other benchmark here answers "how long does this take", which is only readable beside something. This supplies the something: the primitive a caller would otherwise have used. Eight releases of it: add multiply 1.8.0 99.7x 293.9x 1.9.0 99.7x 288.2x 2.0.0 82.8x 248.8x 2.0.1 83.2x 248.3x 2.0.2 83.8x 246.0x 2.0.3 83.7x 252.9x 2.0.4 83.1x 246.2x 2.0.5 83.6x 249.8x Becoming a value type in 2.0 took about 15% off the price of arbitrary precision, and six releases have held it there. Neither half of that is readable from a column of nanoseconds, which is why the ratio is drawn rather than left to be divided by eye. The number is not expected to be 1 and is not a defect for being large -- a double cannot do this at all. What the section is for is noticing the day it moves. The section derives its ratio from two benchmarks already stored rather than recording a new field, so there is no schema change, no history to rewrite, and an entry gathered before the pair existed simply draws as not measured. Three things decide how the number should be read, and all three are in the remarks. The loops exist because a single operation over operands that do not change is loop-invariant and the JIT hoists it out, leaving a double method indistinguishable from an empty one and a ratio against it meaningless. The loop's own cost is paid by both sides and pulls the ratio toward 1, so a ratio is a floor rather than the whole cost. And both loops accumulate rather than compound, because this type carries as many digits as the arithmetic produces and a compounding chain would measure that growth instead of the operation. The backfill now measures published packages rather than each tag's own worktree. Both were defensible until a benchmark was added: a tag predating one cannot run it, so this section could only ever have started from the next release, and the 1.x points above would not exist. Packages give it the same history as everything else, time every version by identical code, and match how the other two repositories backfill. Two consequences follow. ConstructionBenchmarks measures through an internal factory, which a package does not expose, so it is left out of those builds and its panel draws from the working copy onward. And ingest gained the guard the other two already had, refusing a run where every row reads NA rather than recording a release with nothing under it -- measuring packages is exactly where that arises. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017jrnV7N94UGL8fDRRE8Xt8 --- .github/workflows/benchmark-history.yml | 97 +-- Directory.Packages.props | 5 + .../AbstractionCostBenchmarks.cs | 160 +++++ .../PreciseNumber.Benchmarks.csproj | 28 +- PreciseNumber.Benchmarks/README.md | 33 + docs/benchmarks/history.json | 576 ++++++++++-------- docs/benchmarks/performance-dark.svg | 185 +++--- docs/benchmarks/performance.svg | 185 +++--- scripts/benchmark-history.cs | 125 +++- 9 files changed, 906 insertions(+), 488 deletions(-) create mode 100644 PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs diff --git a/.github/workflows/benchmark-history.yml b/.github/workflows/benchmark-history.yml index 4c90f7d..deb6195 100644 --- a/.github/workflows/benchmark-history.yml +++ b/.github/workflows/benchmark-history.yml @@ -5,8 +5,15 @@ name: Benchmark History # which exists to get a full ad-hoc run on demand and publishes nothing. # # Two ways in: -# * a published release, which measures that version and adds one point; -# * a manual dispatch listing refs, which measures each of them in ONE job and backfills. +# * a published release, which measures that version from its own tag and adds one point; +# * a manual dispatch listing versions, which measures each published package in ONE job. +# +# A backfill measures packages rather than tags. Both are defensible until a benchmark is added: a +# tag predating one cannot run it, so a new measurement could only ever start from the next release, +# and AbstractionCostBenchmarks would have had no history at all. Measuring the published packages +# with today's sources gives it the same history as everything else, and times every version by +# identical code besides. ktsu.SignificantNumber and ktsu.Semantics do the same, so the three +# charts are built the same way. # # The backfill running as a single job is the point rather than an optimisation. Separate runs land # on different CI hosts, and the difference between an x86-64-v3 and a v4 runner is larger than @@ -18,15 +25,10 @@ on: types: [published] workflow_dispatch: inputs: - refs: - description: "Space-separated refs to backfill, oldest first (tags, branches, or SHAs)" - required: false - default: "cd9a8227793bbd6ea791be7f6c0579ae1242eec5 v2.0.0 v2.0.1 v2.0.2 v2.0.3 v2.0.4" - type: string - labels: - description: "Optional space-separated version labels matching refs, when a ref is not a version" + versions: + description: "Space-separated released versions to backfill, oldest first" required: false - default: "1.9.0 2.0.0 2.0.1 2.0.2 2.0.3 2.0.4" + default: "1.8.0 1.9.0 2.0.0 2.0.1 2.0.2 2.0.3 2.0.4 2.0.5" type: string permissions: @@ -52,6 +54,7 @@ env: *ConstructionBenchmarks.Sanitizing *TextBenchmarks.Parse *ConversionBenchmarks.ToDouble + *AbstractionCostBenchmarks.* # Short runs: three iterations is enough for a trend line, and a release should not tie up a # runner for half an hour. benchmarks.yml is still there for a full-length run. BENCHMARK_JOB: short @@ -121,51 +124,55 @@ jobs: git worktree remove --force "$work" - - name: Measure each backfill ref + - name: Measure each backfill version if: github.event_name == 'workflow_dispatch' shell: bash env: - REFS: ${{ inputs.refs }} - LABELS: ${{ inputs.labels }} + VERSIONS: ${{ inputs.versions }} BASELINE_NS: ${{ steps.baseline.outputs.ns }} run: | set -euo pipefail - read -ra refs <<< "$REFS" - read -ra labels <<< "$LABELS" - - for index in "${!refs[@]}"; do - ref="${refs[$index]}" - label="${labels[$index]:-${ref#v}}" - work="${RUNNER_TEMP}/bench-$label" - - echo "::group::$label ($ref)" - rm -rf "$work" - git worktree add --detach "$work" "$ref" - - if [ ! -f "$work/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj" ]; then - echo "::warning::$ref has no benchmark project; skipping" - git worktree remove --force "$work" + read -ra versions <<< "$VERSIONS" + + for version in "${versions[@]}"; do + echo "::group::$version" + # Through the environment rather than a -p: switch, because BenchmarkDotNet generates + # and builds a project of its own per run, which a property passed on the command line + # does not reach. MSBuild reads environment variables as properties in every project. + # + # A version whose API the current benchmarks cannot express is reported and skipped, + # rather than failing the whole backfill after the ones before it have been measured. + if ! BenchmarkAgainstVersion="$version" dotnet run -c Release --project PreciseNumber.Benchmarks -- \ + --filter $HEADLINE_FILTER \ + --job "$BENCHMARK_JOB" \ + --artifacts "$GITHUB_WORKSPACE/$RUNS/$version"; then + echo "::warning::$version could not be benchmarked by the current suite; skipping" echo "::endgroup::" continue fi - # Each ref is measured by its own benchmark sources. Between 2.0.0 and now those - # sources are unchanged, so this compares library versions rather than harnesses. - (cd "$work" && dotnet run -c Release --project PreciseNumber.Benchmarks -- \ - --filter $HEADLINE_FILTER \ - --job "$BENCHMARK_JOB" \ - --artifacts "$GITHUB_WORKSPACE/$RUNS/$label") - - dotnet run scripts/benchmark-history.cs -- ingest \ - --history "$HISTORY" \ - --results "$RUNS/$label" \ - --version "$label" \ - --commit "$(git rev-parse --short "$ref^{commit}")" \ - --date "$(git log -1 --format=%cs "$ref")" \ - --run-id "${{ github.run_id }}" \ - --baseline-ns "$BASELINE_NS" - - git worktree remove --force "$work" + tag="v$version" + commit="" + date="" + if git rev-parse -q --verify "$tag^{commit}" >/dev/null; then + commit="$(git rev-parse --short "$tag^{commit}")" + date="$(git log -1 --format=%cs "$tag")" + fi + + # Skipped here too, and for the same reason: a package can build against these + # benchmarks and still throw from every one of them at run time, which BenchmarkDotNet + # reports as a table of NA rather than as a failure. Ingest refuses such a run, and + # the backfill carries on to the next version. + if ! dotnet run scripts/benchmark-history.cs -- ingest \ + --history "$HISTORY" \ + --results "$RUNS/$version" \ + --version "$version" \ + --commit "$commit" \ + --date "$date" \ + --run-id "${{ github.run_id }}" \ + --baseline-ns "$BASELINE_NS"; then + echo "::warning::$version produced no usable measurement; skipping" + fi echo "::endgroup::" done diff --git a/Directory.Packages.props b/Directory.Packages.props index be4d333..dbc7dfc 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,5 +6,10 @@ + + diff --git a/PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs b/PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs new file mode 100644 index 0000000..1d3ecc9 --- /dev/null +++ b/PreciseNumber.Benchmarks/AbstractionCostBenchmarks.cs @@ -0,0 +1,160 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.PreciseNumber.Benchmarks; + +using System.Globalization; + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; + +/// +/// Measures what this type costs against the same arithmetic on a bare . +/// +/// +/// +/// Every other class here answers "how long does this operation take", which is only readable next +/// to something. This one supplies the something: the primitive a caller would otherwise have +/// used. The same class, with the same loops and the same methodology, is in ktsu.SignificantNumber +/// and ktsu.Semantics, so the three answers are comparable with each other as well as with +/// . +/// +/// +/// The bare method is the BenchmarkDotNet baseline, so the answer is the Ratio column rather +/// than two rows divided by hand. Unlike the wrapper types built on this one, a ratio here is not +/// expected to be 1.00 and is not a defect when it is not: arbitrary precision is a cost paid for +/// something a cannot do at all. What the number is for is watching that cost +/// across releases. +/// +/// +/// Why these are loops. A single operation over operands that do not change is +/// loop-invariant, and the JIT hoists it out of the measurement entirely — for +/// that leaves a method indistinguishable from an empty one, and a ratio +/// against an empty method means nothing. Here each iteration feeds the next, so there is nothing +/// to hoist and both sides are measurable. +/// +/// +/// Which way the loop biases the answer. Both sides pay the same counter and branch, and it +/// is a dependency chain, so most of that overlaps the arithmetic; whatever does not is added +/// equally to numerator and denominator and pulls the ratio toward 1.00. A ratio here is therefore +/// a floor on the real cost rather than the whole of it. +/// +/// +/// Why the operands stay bounded, and why they are short. This type carries as many digits +/// as the arithmetic produces, so a chain that compounded its operand would measure that growth +/// rather than the operation; both loops accumulate instead. The operands are also chosen to be +/// values a can hold, so the two sides are doing the same arithmetic on the +/// same numbers rather than being handed different problems. How the cost grows with digits is a +/// different question, and answers it across its +/// Digits axis. +/// +/// +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class AbstractionCostBenchmarks +{ + /// + /// Operations per invocation. Enough that the loop's own cost is a small share of the work, + /// few enough that the arbitrary-precision side still finishes an iteration promptly. + /// + private const int Operations = 256; + + private const string SeedText = "1234.5678901234"; + private const string StepText = "0.0009765625"; + private const string OtherText = "3.14159265358979"; + + private double bareSeed; + private double bareStep; + private double bareOther; + + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. The backfill measures + // those releases too, so the file has to compile against both shapes. + private PreciseNumber preciseSeed = default!; + private PreciseNumber preciseStep = default!; + private PreciseNumber preciseOther = default!; + + /// + /// Prepares the operands, parsed from the same text on both sides. + /// + [GlobalSetup] + public void Setup() + { + bareSeed = double.Parse(SeedText, CultureInfo.InvariantCulture); + bareStep = double.Parse(StepText, CultureInfo.InvariantCulture); + bareOther = double.Parse(OtherText, CultureInfo.InvariantCulture); + + preciseSeed = PreciseNumber.Parse(SeedText, CultureInfo.InvariantCulture); + preciseStep = PreciseNumber.Parse(StepText, CultureInfo.InvariantCulture); + preciseOther = PreciseNumber.Parse(OtherText, CultureInfo.InvariantCulture); + } + + /// Adds along a chain, on a bare double. + /// The accumulated value. + [BenchmarkCategory("Add")] + [Benchmark(Baseline = true, OperationsPerInvoke = Operations)] + public double BareAdd() + { + double accumulator = bareSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += bareStep; + } + + return accumulator; + } + + /// Adds along the same chain, on this type. + /// The accumulated value. + [BenchmarkCategory("Add")] + [Benchmark(OperationsPerInvoke = Operations)] + public PreciseNumber PreciseAdd() + { + PreciseNumber accumulator = preciseSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += preciseStep; + } + + return accumulator; + } + + /// Multiplies and accumulates, on a bare double. + /// The accumulated value. + [BenchmarkCategory("Multiply")] + [Benchmark(Baseline = true, OperationsPerInvoke = Operations)] + public double BareMultiply() + { + double accumulator = 0d; + double value = bareSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += value * bareOther; + value += bareStep; + } + + return accumulator; + } + + /// Multiplies and accumulates over the same values, on this type. + /// The accumulated value. + [BenchmarkCategory("Multiply")] + [Benchmark(OperationsPerInvoke = Operations)] + public PreciseNumber PreciseMultiply() + { + PreciseNumber accumulator = PreciseNumber.Zero; + PreciseNumber value = preciseSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += value * preciseOther; + value += preciseStep; + } + + return accumulator; + } +} diff --git a/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj b/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj index c6aa6ec..fca1d25 100644 --- a/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj +++ b/PreciseNumber.Benchmarks/PreciseNumber.Benchmarks.csproj @@ -29,8 +29,34 @@ - + + + + + + + + + + diff --git a/PreciseNumber.Benchmarks/README.md b/PreciseNumber.Benchmarks/README.md index f9dd876..62e42ec 100644 --- a/PreciseNumber.Benchmarks/README.md +++ b/PreciseNumber.Benchmarks/README.md @@ -50,6 +50,39 @@ each digit separately looks fine at 8 digits and falls apart at 200. Reading a t apart from operands in the same decade, because aligning two exponents is its own cost, distinct from the size of the operands. +## What this type costs against a bare double + +`AbstractionCostBenchmarks` is the one benchmark here whose answer is a ratio rather than a +duration. Every other class says how long an operation takes, which is only readable beside +something; this supplies the something — the primitive a caller would otherwise have used. + +The same class, with the same loops and the same methodology, is in `ktsu.SignificantNumber` and +`ktsu.Semantics`, so the three libraries answer one question the same way and their answers are +comparable with each other as well as with `double`. + +| release | `Add` | `Multiply` | +|---|---|---| +| 1.8.0 | 99.7× | 293.9× | +| 1.9.0 | 99.7× | 288.2× | +| 2.0.0 | 82.8× | 248.8× | +| 2.0.5 | 83.6× | 249.8× | + +Becoming a value type in 2.0 took about 15% off the price of arbitrary precision, and six releases +have held it there. **The ratio is not expected to be 1 and is not a defect for being large** — a +`double` cannot do this at all. What the chart's third section is for is noticing the day it moves. + +Three things decide how the number should be read: + +- **These are loops.** A single operation over operands that do not change is loop-invariant and + the JIT hoists it out, which would leave the `double` side indistinguishable from an empty method + and the ratio meaningless. Each iteration feeds the next, so there is nothing to hoist. +- **The loop's own cost biases toward 1**, being paid by both sides, so a ratio is a floor on the + real cost rather than the whole of it. +- **Both loops accumulate rather than compound**, because this type carries as many digits as the + arithmetic produces and a compounding chain would measure that growth instead of the operation. + How the cost grows with digits is a different question, and `ArithmeticBenchmarks` answers it + across the `Digits` axis. + ## Reading the results Allocation is reported next to time. Both matter here, and they trade against each other: every diff --git a/docs/benchmarks/history.json b/docs/benchmarks/history.json index 2d37639..6d70308 100644 --- a/docs/benchmarks/history.json +++ b/docs/benchmarks/history.json @@ -7,96 +7,106 @@ "date": "2026-09-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9865, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3522, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 98.3125, + "allocatedBytes": 72 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 397.4461, + "allocatedBytes": 243 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 123.1859, + "meanNs": 116.6638, "allocatedBytes": 104 }, "Digits=30": { - "meanNs": 164.7113, + "meanNs": 161.361, "allocatedBytes": 120 }, "Digits=200": { - "meanNs": 402.6075, + "meanNs": 420.159, "allocatedBytes": 264 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 794.4781, + "meanNs": 817.6077, "allocatedBytes": 264 }, "Digits=30": { - "meanNs": 889.3647, + "meanNs": 858.7159, "allocatedBytes": 312 }, "Digits=200": { - "meanNs": 995.529, + "meanNs": 958.3089, "allocatedBytes": 456 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 81.9846, + "meanNs": 87.7939, "allocatedBytes": 72 }, "Digits=30": { - "meanNs": 184.0505, + "meanNs": 183.6178, "allocatedBytes": 96 }, "Digits=200": { - "meanNs": 1050.0351, + "meanNs": 1057.627, "allocatedBytes": 232 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.5, + "meanNs": 4.6545, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 4.7745, + "meanNs": 4.7675, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 5.1859, + "meanNs": 4.7959, "allocatedBytes": 0 } }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 35.0957, - "allocatedBytes": 40 - }, - "Digits=30": { - "meanNs": 76.4736, - "allocatedBytes": 40 - }, - "Digits=200": { - "meanNs": 254.6939, - "allocatedBytes": 40 - } - }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 20.149, + "meanNs": 21.0712, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 199.5932, + "meanNs": 207.9911, "allocatedBytes": 40 }, "Digits=30": { - "meanNs": 480.1194, + "meanNs": 488.4566, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 2534.6838, + "meanNs": 2561.4915, "allocatedBytes": 152 } } @@ -108,96 +118,106 @@ "date": "2026-09-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9939, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.356, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 99.0794, + "allocatedBytes": 72 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 390.7421, + "allocatedBytes": 243 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 114.0296, + "meanNs": 121.8675, "allocatedBytes": 104 }, "Digits=30": { - "meanNs": 156.3233, + "meanNs": 164.1578, "allocatedBytes": 120 }, "Digits=200": { - "meanNs": 428.4019, + "meanNs": 431.5375, "allocatedBytes": 264 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 403.9122, + "meanNs": 417.5664, "allocatedBytes": 192 }, "Digits=30": { - "meanNs": 613.6075, + "meanNs": 619.2303, "allocatedBytes": 240 }, "Digits=200": { - "meanNs": 2375.4439, + "meanNs": 3484.2076, "allocatedBytes": 568 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 84.6192, + "meanNs": 86.8419, "allocatedBytes": 72 }, "Digits=30": { - "meanNs": 179.2593, + "meanNs": 185.772, "allocatedBytes": 96 }, "Digits=200": { - "meanNs": 1029.5614, + "meanNs": 1051.0365, "allocatedBytes": 232 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.55, + "meanNs": 4.6033, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 4.7625, + "meanNs": 4.6786, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 4.5483, + "meanNs": 4.7639, "allocatedBytes": 0 } }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 35.4251, - "allocatedBytes": 40 - }, - "Digits=30": { - "meanNs": 80.1594, - "allocatedBytes": 40 - }, - "Digits=200": { - "meanNs": 262.4124, - "allocatedBytes": 40 - } - }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 20.7065, + "meanNs": 20.3028, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 205.5582, + "meanNs": 206.2804, "allocatedBytes": 40 }, "Digits=30": { - "meanNs": 490.8118, + "meanNs": 490.0897, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 2503.218, + "meanNs": 2644.6797, "allocatedBytes": 152 } } @@ -209,96 +229,106 @@ "date": "2026-09-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9959, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3698, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.487, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 340.8541, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 100.4854, + "meanNs": 102.2745, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 145.0197, + "meanNs": 152.0189, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 406.8626, + "meanNs": 406.6596, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 421.971, + "meanNs": 431.6098, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 649.0157, + "meanNs": 625.1701, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2501.2133, + "meanNs": 2410.6448, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 70.0239, + "meanNs": 69.667, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 175.1806, + "meanNs": 169.1872, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1025.2062, + "meanNs": 1025.3157, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 3.1494, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 4.2118, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.589, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.0883, + "meanNs": 3.9819, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 63.0727, + "meanNs": 4.1473, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 242.1056, + "meanNs": 3.5786, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.749, + "meanNs": 5.2676, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 187.9601, + "meanNs": 191.5534, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 479.94, + "meanNs": 482.5276, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2528.7065, + "meanNs": 2563.0209, "allocatedBytes": 112 } } @@ -310,96 +340,106 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9927, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.365, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.6058, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 338.9373, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 100.8056, + "meanNs": 103.9937, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 148.6598, + "meanNs": 147.0839, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 576.7779, + "meanNs": 406.8272, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 432.8415, + "meanNs": 447.1332, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 630.3228, + "meanNs": 653.7294, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2408.9761, + "meanNs": 3460.0593, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 71.7227, + "meanNs": 70.7119, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 168.2777, + "meanNs": 168.7656, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1036.3418, + "meanNs": 1028.2038, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 3.9038, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 4.1531, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 4.2655, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.3621, + "meanNs": 3.6292, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 63.2541, + "meanNs": 4.2232, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 242.4946, + "meanNs": 3.8236, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.0037, + "meanNs": 5.6916, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 186.071, + "meanNs": 191.3243, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 486.0585, + "meanNs": 484.8135, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2492.1256, + "meanNs": 2494.1918, "allocatedBytes": 112 } } @@ -411,96 +451,106 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9882, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3598, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.8552, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 334.529, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 101.1571, + "meanNs": 103.792, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 147.8035, + "meanNs": 150.2509, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 397.9905, + "meanNs": 406.1234, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 421.5628, + "meanNs": 432.5153, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 648.9157, + "meanNs": 643.1053, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2435.5906, + "meanNs": 2408.4417, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 74.7106, + "meanNs": 85.3321, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 169.6895, + "meanNs": 172.561, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1011.6622, + "meanNs": 1021.9672, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.0939, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.8344, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.6637, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 20.3191, + "meanNs": 3.617, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 64.0998, + "meanNs": 3.8082, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 244.0778, + "meanNs": 3.5699, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.3519, + "meanNs": 5.3681, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 185.887, + "meanNs": 193.862, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 489.6898, + "meanNs": 489.4994, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2449.7325, + "meanNs": 2476.3004, "allocatedBytes": 112 } } @@ -512,96 +562,106 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9904, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3589, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.8596, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 343.61, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 106.0063, + "meanNs": 104.5388, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 145.0424, + "meanNs": 150.2235, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 400.2685, + "meanNs": 407.8852, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 423.013, + "meanNs": 423.821, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 641.3537, + "meanNs": 625.2752, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 3414.9285, + "meanNs": 3511.3953, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 67.2843, + "meanNs": 69.7463, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 168.3793, + "meanNs": 169.4646, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1019.9757, + "meanNs": 1016.2448, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.4645, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.8372, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.5919, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.6194, + "meanNs": 3.5734, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 67.0253, + "meanNs": 4.4563, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 247.2198, + "meanNs": 3.9594, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.1391, + "meanNs": 4.8, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 190.2204, + "meanNs": 187.8255, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 485.4024, + "meanNs": 481.0221, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2529.4197, + "meanNs": 2547.2377, "allocatedBytes": 112 } } @@ -613,96 +673,106 @@ "date": "2026-09-15", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9948, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3565, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.7114, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 333.9983, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 103.5012, + "meanNs": 104.7875, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 147.654, + "meanNs": 147.415, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 395.7874, + "meanNs": 408.9774, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 419.0343, + "meanNs": 434.0507, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 627.7278, + "meanNs": 637.4499, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 2440.4735, + "meanNs": 3422.1123, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 69.7283, + "meanNs": 68.155, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 165.934, + "meanNs": 171.208, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1027.7703, + "meanNs": 1026.3715, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 3.5588, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.5576, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.6354, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.7243, + "meanNs": 4.1605, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 63.6485, + "meanNs": 3.5677, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 241.9847, + "meanNs": 4.2016, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 5.3881, + "meanNs": 5.4118, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 189.778, + "meanNs": 189.268, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 477.5123, + "meanNs": 475.3413, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2512.168, + "meanNs": 2369.1519, "allocatedBytes": 112 } } @@ -714,96 +784,106 @@ "date": "2026-09-16", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 454.8992, + "baselineNs": 454.8648, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9912, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3655, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.PreciseAdd": { + "": { + "meanNs": 82.8329, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.PreciseMultiply": { + "": { + "meanNs": 341.1631, + "allocatedBytes": 123 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 104.0341, + "meanNs": 106.8633, "allocatedBytes": 64 }, "Digits=30": { - "meanNs": 144.1285, + "meanNs": 146.4693, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 397.1837, + "meanNs": 408.8275, "allocatedBytes": 224 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 425.7019, + "meanNs": 430.0093, "allocatedBytes": 152 }, "Digits=30": { - "meanNs": 615.669, + "meanNs": 646.074, "allocatedBytes": 200 }, "Digits=200": { - "meanNs": 3372.9288, + "meanNs": 2476.1516, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 68.7651, + "meanNs": 68.9266, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 167.7462, + "meanNs": 173.2572, "allocatedBytes": 56 }, "Digits=200": { - "meanNs": 1010.8013, + "meanNs": 1021.0018, "allocatedBytes": 192 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 4.2163, - "allocatedBytes": 0 - }, - "Digits=30": { - "meanNs": 3.8202, - "allocatedBytes": 0 - }, - "Digits=200": { - "meanNs": 3.5939, - "allocatedBytes": 0 - } - }, - "ConstructionBenchmarks.Sanitizing": { - "Digits=8": { - "meanNs": 19.1214, + "meanNs": 3.5867, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 62.9138, + "meanNs": 3.7923, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 241.3521, + "meanNs": 4.1539, "allocatedBytes": 0 } }, "ConversionBenchmarks.ToDouble": { "": { - "meanNs": 4.8894, + "meanNs": 4.7214, "allocatedBytes": 0 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 194.2737, + "meanNs": 196.5275, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 476.8454, + "meanNs": 481.0981, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2442.8716, + "meanNs": 2521.0782, "allocatedBytes": 112 } } diff --git a/docs/benchmarks/performance-dark.svg b/docs/benchmarks/performance-dark.svg index 28fc8de..479cb33 100644 --- a/docs/benchmarks/performance-dark.svg +++ b/docs/benchmarks/performance-dark.svg @@ -1,4 +1,4 @@ - + - + PreciseNumber performance by release 8 releases · newest 2.0.5 · 2026-09-16 @@ -68,18 +68,7 @@ 0 B 0 B Construct (30 digits) - - - - - - - - - - -0 B -40 B +not measured Parse (30 digits) @@ -111,95 +100,113 @@ Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. Add (30 digits) - - - - - - - - - -0.317× -0.362× + + + + + + + + + +0.322× +0.355× Multiply (30 digits) - - - - - - - - - -0.369× -0.405× + + + + + + + + + +0.381× +0.404× Divide (30 digits) - + - - - - - - - -1.35× -1.96× + + + + + + + +1.42× +1.89× CompareTo (30 digits) - + - - - - - - - -0.0084× + + + + + + + +0.0083× 0.0105× Construct (30 digits) - - - - - - - - - - -0.138× -0.168× +not measured Parse (30 digits) - - + + - - + + - - - -1.05× -1.06× + + + +1.06× +1.07× ToDouble - - - - - - - - - -0.0107× -0.0443× -releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 -Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. + + + + + + + + + +0.0104× +0.0463× + +Cost over the same arithmetic on a bare double +Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put. +Add + + + + + + + + + + +83.6× +99.7× +Multiply + + + + + + + + + + +250× +294× +releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 +Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. diff --git a/docs/benchmarks/performance.svg b/docs/benchmarks/performance.svg index b417420..77bb00f 100644 --- a/docs/benchmarks/performance.svg +++ b/docs/benchmarks/performance.svg @@ -1,4 +1,4 @@ - + - + PreciseNumber performance by release 8 releases · newest 2.0.5 · 2026-09-16 @@ -68,18 +68,7 @@ 0 B 0 B Construct (30 digits) - - - - - - - - - - -0 B -40 B +not measured Parse (30 digits) @@ -111,95 +100,113 @@ Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. Add (30 digits) - - - - - - - - - -0.317× -0.362× + + + + + + + + + +0.322× +0.355× Multiply (30 digits) - - - - - - - - - -0.369× -0.405× + + + + + + + + + +0.381× +0.404× Divide (30 digits) - + - - - - - - - -1.35× -1.96× + + + + + + + +1.42× +1.89× CompareTo (30 digits) - + - - - - - - - -0.0084× + + + + + + + +0.0083× 0.0105× Construct (30 digits) - - - - - - - - - - -0.138× -0.168× +not measured Parse (30 digits) - - + + - - + + - - - -1.05× -1.06× + + + +1.06× +1.07× ToDouble - - - - - - - - - -0.0107× -0.0443× -releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 -Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. + + + + + + + + + +0.0104× +0.0463× + +Cost over the same arithmetic on a bare double +Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put. +Add + + + + + + + + + + +83.6× +99.7× +Multiply + + + + + + + + + + +250× +294× +releases, oldest to newest: 1.8.0 → 1.9.0 → 2.0.0 → 2.0.1 → 2.0.2 → 2.0.3 → 2.0.4 → 2.0.5 +Measured on Intel Xeon Processor 2.80GHz. Full tables: PreciseNumber.Benchmarks. diff --git a/scripts/benchmark-history.cs b/scripts/benchmark-history.cs index 408895d..8192873 100644 --- a/scripts/benchmark-history.cs +++ b/scripts/benchmark-history.cs @@ -49,12 +49,39 @@ private static readonly (string Key, string? Parameters, string Label)[] Headlin /// private static readonly Dictionary Themes = new(StringComparer.Ordinal) { - ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834"), - ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926"), + ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834", "#2e8b57"), + ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926", "#3faa71"), }; private sealed record Theme( - string Surface, string Ink, string Muted, string Grid, string Alloc, string Time); + string Surface, string Ink, string Muted, string Grid, string Alloc, string Time, string Cost); + + /// Which of the three things a section draws. + private enum Measure + { + /// Bytes allocated per operation, as stored. + Allocation, + + /// Time, divided by the reference workload from the same job. + Time, + + /// Time, divided by the paired bare-double benchmark from the same entry. + Cost, + } + + /// + /// The paired benchmarks the cost section draws, as (measured, baseline, label). + /// + /// + /// Both halves are stored like any other benchmark; the ratio is computed here rather than + /// recorded, so an entry gathered before this section existed still draws once its run + /// includes the pair, and no history has to be rewritten to change what the section shows. + /// + private static readonly (string Key, string Baseline, string Label)[] CostHeadline = + [ + ("AbstractionCostBenchmarks.PreciseAdd", "AbstractionCostBenchmarks.BareAdd", "Add"), + ("AbstractionCostBenchmarks.PreciseMultiply", "AbstractionCostBenchmarks.BareMultiply", "Multiply"), + ]; internal static int Run(string[] args) { @@ -161,6 +188,19 @@ private static int Ingest(Dictionary options) } string version = Required(options, "version"); + JsonObject benchmarks = Benchmarks(measured); + if (benchmarks.Count == 0) + { + // Reports with every row reading NA: the harness built and ran, and each benchmark + // threw. An older package whose API these benchmarks compile against but whose + // behaviour they cannot exercise does exactly this. Recording it would put a release + // on the axis with nothing under it, which reads as a release that was measured and + // found to cost nothing. + Console.Error.WriteLine( + $"No benchmark in {resultsDirectory} produced a measurement; {version} not recorded"); + return 1; + } + JsonObject record = new() { ["version"] = version, @@ -170,7 +210,7 @@ private static int Ingest(Dictionary options) ["runtime"] = runtime, ["baselineNs"] = baseline, ["runId"] = Optional(options, "run-id"), - ["benchmarks"] = Benchmarks(measured), + ["benchmarks"] = benchmarks, }; string historyPath = Required(options, "history"); @@ -390,16 +430,17 @@ private static string Draw(JsonArray entries, Theme theme) string[] labels = [.. entries.Select(entry => entry!["version"]?.GetValue() ?? "?")]; int width = Left + (Columns * CellWidth) + 24; int rows = (Headline.Length + Columns - 1) / Columns; - int height = 72 + (((34 + (rows * CellHeight)) * 2) + 54); + int costRows = (CostHeadline.Length + Columns - 1) / Columns; + int height = 72 + ((34 + (rows * CellHeight)) * 2) + 34 + (costRows * CellHeight) + 54; StringBuilder svg = new(); Preamble(svg, theme, width, height, entries); int y = 72; - foreach (bool isTime in (bool[])[false, true]) + foreach (Measure measure in (Measure[])[Measure.Allocation, Measure.Time, Measure.Cost]) { - Section(svg, theme, entries, labels.Length, y, isTime); - y += 34 + (rows * CellHeight); + Section(svg, theme, entries, labels.Length, y, measure); + y += 34 + ((measure == Measure.Cost ? costRows : rows) * CellHeight); } Footer(svg, entries, labels, y - 4); @@ -429,20 +470,53 @@ private static void Preamble(StringBuilder svg, Theme theme, int width, int heig svg.AppendLine(CultureInfo.InvariantCulture, $"""{entries.Count} releases · newest {Escape(latest["version"]?.GetValue() ?? "?")}{suffix}"""); } - private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, bool isTime) + private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, Measure measure) { - string colour = isTime ? theme.Time : theme.Alloc; - string title = isTime - ? "Time, as a multiple of a fixed reference workload" - : "Allocated bytes per operation"; - string note = isTime - ? "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster." - : "Deterministic: the same code allocates the same bytes on any machine."; + string colour = measure switch + { + Measure.Time => theme.Time, + Measure.Cost => theme.Cost, + _ => theme.Alloc, + }; + string title = measure switch + { + Measure.Time => "Time, as a multiple of a fixed reference workload", + Measure.Cost => "Cost over the same arithmetic on a bare double", + _ => "Allocated bytes per operation", + }; + string note = measure switch + { + Measure.Time => "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster.", + Measure.Cost => "Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put.", + _ => "Deterministic: the same code allocates the same bytes on any machine.", + }; svg.AppendLine(CultureInfo.InvariantCulture, $""""""); svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(title)}"""); svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(note)}"""); + if (measure == Measure.Cost) + { + for (int position = 0; position < CostHeadline.Length; position++) + { + (string key, string baseline, string label) = CostHeadline[position]; + double?[] values = [.. entries.Select(entry => Cost(entry!, key, baseline))]; + Panel( + svg, + Left + (position % Columns * CellWidth), + y + 26 + (position / Columns * CellHeight), + label, + points, + values, + true, + colour, + theme); + } + + return; + } + + bool isTime = measure == Measure.Time; for (int position = 0; position < Headline.Length; position++) { (string key, string? parameters, string label) = Headline[position]; @@ -460,6 +534,25 @@ private static void Section(StringBuilder svg, Theme theme, JsonArray entries, i } } + /// + /// One benchmark's mean divided by the mean of the bare-double benchmark beside it. + /// + /// + /// Both were measured in the same job on the same machine, so unlike the time section this + /// needs no reference workload to be comparable across runs: the denominator is the reference. + /// + private static double? Cost(JsonNode entry, string key, string baseline) + { + double? measured = Mean(entry, key); + double? divisor = Mean(entry, baseline); + return measured is not null && divisor is > 0 ? measured / divisor : null; + } + + private static double? Mean(JsonNode entry, string key) => + entry["benchmarks"]?[key] is JsonObject cases && cases.Count > 0 + ? cases.First().Value?["meanNs"]?.GetValue() + : null; + private static double? Value(JsonNode entry, string key, string? parameters, bool isTime) { if (entry["benchmarks"]?[key] is not JsonObject cases || cases.Count == 0)