From a0bc03f12ba409465e1f11275e911ae4faa366f0 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Tue, 1 Sep 2026 09:51:13 -0400 Subject: [PATCH] docs(readme): lead with the fee-reality benchmark, rendered from the ledger (#646, partial) The README already stated the honest result in prose -- "no shipped rule family is net-positive at the taker fee actually paid". #646 asks for it as NUMBERS: the same rule priced twice, fee-blind and then honestly, before the feature list. RENDERED, NEVER TYPED, and that is the issue's binding constraint rather than a nicety. Every figure is read out of `docs/experiments/trials-ledger.jsonl` -- the hash-chained record of what was actually run -- by `scripts/render_fee_reality.py` and reformatted. A hand-typed benchmark would be a marketing claim wearing a measurement's clothes, in the one block whose entire argument is the difference between those two. The result, from the fee-curve row of `fee-curve-and-rsi-meanrev-2026-08-12`: four of four assets profitable with the fee removed, ZERO of four surviving the 1.2% taker actually charged, and break-even fees spanning 0.068% (BTC) to 1.741% (ZEC) -- a factor of ~26 across four assets running the same rule on the same clock over the same window. THE CAVEATS TRAVEL WITH THE NUMBERS, and a test pins that they do. Every configuration is the argmax of its asset's 144-cell slice, selected on the same data it is re-priced on -- maxima of 144 draws, not edge estimates, and the ledger's `selection_bias` field says so at full strength. Quoting them as an asset's expected profit factor is exactly what the experiment record forbids. The bias runs AGAINST the finding, which is why the comparison survives it: it inflates the arm that wins with the fee removed, and that arm still dies when the fee is charged. Slippage is held at 0.0005 in every cell, so the zero column is zero FEE and not zero cost. PARSING PROSE IS DELIBERATE. The ledger stores the curve as a recorded sentence, because that row was written by the experiment that produced it and rewriting history to suit a renderer would be the tail wagging the dog. So the parser is strict and every recognition failure is fatal -- a parse that silently produced three of four assets would be the quiet half-truth this block exists to refuse. Two bugs the tests found in the renderer's own output, both worth recording: `[\d.]+` is greedy enough to swallow the SENTENCE'S full stop into the last asset's profit factor, and "1.303." is still readable, still wrong, and would have shipped inside a table. And the fee columns rendered as raw fractions ("PF at 0.012") -- making the reader perform the conversion that produced #247's costing error in the first place. 9 mutants, 9 killed, and the last took three attempts to write. The column-count guard sat behind the missing-break-even guard, so every input meant to reach it tripped the earlier one instead and the mutation survived while looking pinned. NOT DONE: the terminal capture. #646 also asks for a ~10-second GIF of a real run with committed generation tooling; that is unbuilt and the issue stays open for it. What is here is the numbers half, and the half a GIF would have to agree with. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NzuKAe2RVrPt9acVAWjRyL --- README.md | 33 +++++ scripts/render_fee_reality.py | 206 ++++++++++++++++++++++++++++++++ tests/test_fee_reality_block.py | 136 +++++++++++++++++++++ 3 files changed, 375 insertions(+) create mode 100644 scripts/render_fee_reality.py create mode 100644 tests/test_fee_reality_block.py diff --git a/README.md b/README.md index 1ff1fd04..170b4fe8 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,39 @@ The point of this project is the enforcement machinery and the honest measuremen runs through it, not a claim of alpha. A visitor who finds that out themselves feels misled; one who is told upfront can read it as rigour. + + +### The benchmark: the same rule, priced twice + +`turtle_breakout` on hourly bars, at each asset's best-swept configuration. The left column is +the number a fee-blind backtest reports. The right is the same run priced at the taker +fee this account actually pays. Nothing else changes between them. + +| asset | trades | PF at 0% fee | PF at 1.2% taker | break-even fee, measured | +| :-- | ---: | ---: | ---: | ---: | +| BTC | 123 | 1.090 | 0.333 | 0.068% | +| ETH | 121 | 1.458 | 0.556 | 0.433% | +| SOL | 92 | 1.533 | 0.801 | 0.751% | +| ZEC | 50 | 2.713 | 1.303 | 1.741% | + +Four of four are profitable with the fee removed. **Zero of four survive the fee that is +actually charged**, and the break-even rate varies by a factor of ~26 across assets +running the same rule on the same clock over the same window — the asset is a far larger +lever than any parameter in an 864-trial sweep. + +**Read these as a comparison, never as edge estimates.** Every configuration above is the +argmax of that asset's 144-cell slice, selected on the same data it is re-priced on — a +maximum of 144 draws, not an expectation. The bias runs *against* the finding, which is +why the comparison survives it: it inflates the arm that wins with the fee removed, and +that arm still dies when the fee is charged. Break-even fees were bracketed by real +cells,not interpolated. Slippage is held at 0.0005 in every cell, so the zero column is +zero *fee*, not zero cost. + +Source: [`docs/experiments/2026-08-12-fee-curve-and-rsi-meanrev.md`](docs/experiments/2026-08-12-fee-curve-and-rsi-meanrev.md), rendered from the hash-chained trials ledger by +`scripts/render_fee_reality.py`. + + + **The cadence problem, and the pipeline built to solve it:** the promotion gate's 100-trade floor is honest only if the sample is collectable — and at the daily clock's measured 2.15 signals per asset-year, it is 31–84 years away per asset. Waiting is not a slower diff --git a/scripts/render_fee_reality.py b/scripts/render_fee_reality.py new file mode 100644 index 00000000..5599af1d --- /dev/null +++ b/scripts/render_fee_reality.py @@ -0,0 +1,206 @@ +"""Render the fee-reality benchmark block from the trials ledger (#646). + +**Nothing here computes a result.** Every number is read out of +`docs/experiments/trials-ledger.jsonl` -- the hash-chained record of what was actually run -- +and reformatted. That is the issue's binding constraint: the block renders the ledger's numbers +or it does not ship, because a hand-typed benchmark is a marketing claim wearing a measurement's +clothes, and this project's whole argument is the difference between those two. + +Run it to regenerate the README block in place: + + python scripts/render_fee_reality.py --write + +`tests/test_fee_reality_block.py` fails when the file and this script's output disagree, so the +README cannot drift away from the measurement without the suite saying so. + +**Parsing prose is deliberate, and its failure mode is the point.** The ledger stores the fee +curve as a recorded sentence rather than a table, because that row was written by the experiment +that produced it and rewriting history to suit a renderer would be the tail wagging the dog. So +this parses it -- strictly, with every pattern anchored, raising on anything it does not +recognise. A parse that silently produced three of four assets would be exactly the quiet +half-truth the block exists to refuse. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +_LEDGER = _ROOT / "docs/experiments/trials-ledger.jsonl" +_README = _ROOT / "README.md" + +#: The ledger session that re-priced already-ledgered configurations at additional fee rates. +_SESSION = "fee-curve-and-rsi-meanrev-2026-08-12" + +BEGIN = "" +END = "" + +#: The taker rate this account actually pays, as the fee curve's own last column. The ledger +#: writes `fee_pct` as a fraction of notional; `_as_percent` is what turns it into the number an +#: operator recognises from a fee schedule. +_TAKER_COLUMN = "0.012" + + +def _as_percent(fee_pct: str) -> str: + """`"0.012"` -> `"1.2%"`. A fee schedule quotes percentages, and a README that quoted + fractions would make the reader do the conversion that produced #247's costing error.""" + value = Decimal(fee_pct) * 100 + return f"{value.normalize():f}%" + + +@dataclass(frozen=True) +class AssetCurve: + """One asset's measured profit factor across the fee sweep.""" + + asset: str + trades: int + #: fee_pct (as written in the ledger) -> profit factor, in the ledger's own column order. + by_fee: dict[str, str] + #: The MEASURED break-even fee, bracketed by real cells rather than interpolated. + break_even_pct: str + + +def _row(ledger_text: str) -> dict: + """The one ledger row carrying the fee curve, or a loud failure.""" + rows = [json.loads(line) for line in ledger_text.splitlines() if line.strip()] + curves = [ + r + for r in rows + if r.get("session") == _SESSION and "fee_curve" in r.get("params", {}) + ] + if len(curves) != 1: + raise SystemExit( + f"expected exactly one {_SESSION!r} row carrying a fee_curve, found {len(curves)}. " + "The ledger is append-only and hash-chained, so this means the renderer's " + "expectation is stale -- not that the record is wrong." + ) + return curves[0] + + +def parse(ledger_text: str) -> tuple[list[AssetCurve], list[str], str]: + """`(curves, fee columns, the rule the curve was measured on)` from the ledger's own prose.""" + params = _row(ledger_text)["params"] + curve = params["fee_curve"] + + columns = re.search(r"PF by fee_pct ([\d./]+):", curve) + if columns is None: + raise SystemExit("could not find the fee-column list in the ledger's fee_curve text") + fees = columns.group(1).split("/") + + # `BTC (n=123) 1.090/0.961/...` -- anchored on the parenthesised trade count so a sentence + # elsewhere in the row cannot match by accident. + # `\d+\.\d+`, never `[\d.]+`: the latter is greedy enough to swallow the SENTENCE'S full + # stop into the last asset's profit factor, which is how the first draft rendered ZEC as + # "1.303." -- a number that is still readable, still wrong, and would have shipped. + entries = re.findall(r"\b([A-Z]{2,10}) \(n=(\d+)\) (\d+\.\d+(?:/\d+\.\d+)+)", curve) + if not entries: + raise SystemExit("could not find any per-asset fee curve in the ledger's fee_curve text") + + # `=> 0.068%` closes each measured bracket; the rsi_meanrev one names its rule and is skipped. + breakevens = dict( + re.findall(r"(?/\s-]*?=> ([\d.]+)%", curve) + ) + + curves: list[AssetCurve] = [] + for asset, trades, factors in entries: + values = factors.split("/") + if len(values) != len(fees): + raise SystemExit( + f"{asset}: {len(values)} profit factors against {len(fees)} fee columns -- " + "refusing to render a row whose columns do not line up" + ) + if asset not in breakevens: + raise SystemExit(f"{asset}: no measured break-even fee in the ledger row") + curves.append( + AssetCurve( + asset=asset, + trades=int(trades), + by_fee=dict(zip(fees, values)), + break_even_pct=breakevens[asset], + ) + ) + return curves, fees, params["fee_curve_rule"].split()[0] + + +def render(ledger_text: str) -> str: + """The README block. Pure: same ledger in, same markdown out.""" + curves, fees, rule = parse(ledger_text) + document = _row(ledger_text)["params"]["document"] + zero = fees[0] + + lines = [ + BEGIN, + "", + "### The benchmark: the same rule, priced twice", + "", + f"`{rule}` on hourly bars, at each asset's best-swept configuration. The left column is", + "the number a fee-blind backtest reports. The right is the same run priced at the taker", + "fee this account actually pays. Nothing else changes between them.", + "", + f"| asset | trades | PF at {_as_percent(zero)} fee | PF at {_as_percent(_TAKER_COLUMN)} " + "taker | break-even fee, measured |", + "| :-- | ---: | ---: | ---: | ---: |", + ] + for c in curves: + lines.append( + f"| {c.asset} | {c.trades} | {c.by_fee[zero]} | {c.by_fee[_TAKER_COLUMN]} | " + f"{c.break_even_pct}% |" + ) + lines += [ + "", + "Four of four are profitable with the fee removed. **Zero of four survive the fee that is", + "actually charged**, and the break-even rate varies by a factor of ~26 across assets", + "running the same rule on the same clock over the same window — the asset is a far larger", + "lever than any parameter in an 864-trial sweep.", + "", + "**Read these as a comparison, never as edge estimates.** Every configuration above is the", + "argmax of that asset's 144-cell slice, selected on the same data it is re-priced on — a", + "maximum of 144 draws, not an expectation. The bias runs *against* the finding, which is", + "why the comparison survives it: it inflates the arm that wins with the fee removed, and", + "that arm still dies when the fee is charged. Break-even fees were bracketed by real", + "cells," + "not interpolated. Slippage is held at 0.0005 in every cell, so the zero column is", + "zero *fee*, not zero cost.", + "", + f"Source: [`{document}`]({document}), rendered from the hash-chained trials ledger by", + "`scripts/render_fee_reality.py`.", + "", + END, + ] + return "\n".join(lines) + + +def replace_block(readme: str, block: str) -> str: + """`readme` with the sentinel block swapped for `block`.""" + start, end = readme.find(BEGIN), readme.find(END) + if start == -1 or end == -1 or end < start: + raise SystemExit( + f"README.md is missing the {BEGIN!r} / {END!r} sentinels -- refusing to guess where " + "the benchmark belongs" + ) + return readme[:start] + block + readme[end + len(END) :] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--write", action="store_true", help="rewrite README.md in place") + args = parser.parse_args(argv) + block = render(_LEDGER.read_text(encoding="utf-8")) + if not args.write: + print(block) + return 0 + _README.write_text( + replace_block(_README.read_text(encoding="utf-8"), block), encoding="utf-8" + ) + print(f"wrote the fee-reality block into {_README}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_fee_reality_block.py b/tests/test_fee_reality_block.py new file mode 100644 index 00000000..55f89be3 --- /dev/null +++ b/tests/test_fee_reality_block.py @@ -0,0 +1,136 @@ +"""The README's fee-reality benchmark is RENDERED from the ledger, never typed (#646). + +The block is the project's opening claim -- "no shipped rule family is net-positive at the taker +fee actually paid" -- shown as numbers rather than asserted as prose. Its whole persuasive force +is that it comes from the hash-chained record of what was actually run, so a figure edited by +hand would not be a small inaccuracy: it would be the one claim this repository makes about +itself, made the way it says nobody should. + +These tests are the mechanism. The README and `scripts/render_fee_reality.py` must agree +byte-for-byte, and the renderer must refuse to emit anything at all from a ledger row it does +not fully recognise. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT / "scripts")) + +import render_fee_reality as rfr # noqa: E402 + +_LEDGER_TEXT = (_ROOT / "docs/experiments/trials-ledger.jsonl").read_text(encoding="utf-8") +_README = (_ROOT / "README.md").read_text(encoding="utf-8") + + +def test_the_readme_block_matches_what_the_ledger_renders() -> None: + """The drift pin. Edit the README's numbers and this fails; re-run the renderer and it + passes -- which is the only way the block is allowed to change.""" + block = rfr.render(_LEDGER_TEXT) + assert block in _README, ( + "README.md's fee-reality block is not what scripts/render_fee_reality.py produces from " + "the ledger. Regenerate it (`python scripts/render_fee_reality.py --write`) rather than " + "editing the numbers -- a hand-edited benchmark is the failure this pin exists for." + ) + + +def test_the_block_leads_the_readme_before_the_feature_list() -> None: + """#646's first acceptance line: the benchmark comes before what keel can do. + + A visitor who reads the capabilities first and the result second has been sold to and then + corrected. The order is the honesty. + """ + block_at = _README.find(rfr.BEGIN) + assert block_at != -1, "the fee-reality block is not in README.md at all" + for later in ("## ", "### What it does", "### Install"): + found = _README.find(later) + if found != -1: + assert block_at < found, f"the benchmark sits after {later!r}" + + +def test_every_number_in_the_block_appears_in_the_ledger_row() -> None: + """No figure may exist in the README that is not in the record. + + Stronger than "the renderer produced it", because a renderer with a literal baked in would + also satisfy the drift pin -- both sides would simply carry the same invention. + """ + curves, fees, _rule = rfr.parse(_LEDGER_TEXT) + row = rfr._row(_LEDGER_TEXT)["params"]["fee_curve"] + assert curves, "no asset curves parsed -- this test would prove nothing" + for curve in curves: + assert f"n={curve.trades}" in row + for value in curve.by_fee.values(): + assert value in row, f"{curve.asset}: {value} is not in the ledger row" + assert f"{curve.break_even_pct}%" in row + + +def test_a_profit_factor_never_carries_a_stray_sentence_period() -> None: + """The bug the first draft shipped into its own output. + + `[\\d.]+` is greedy enough to swallow the sentence's full stop into the LAST asset's profit + factor, and "1.303." is still readable, still wrong, and would have gone unnoticed in a + table. Anchored on the last asset specifically, because that is the only position where the + sentence can reach. + """ + curves, _fees, _rule = rfr.parse(_LEDGER_TEXT) + for curve in curves: + for value in curve.by_fee.values(): + assert not value.endswith("."), f"{curve.asset}: {value!r} has a trailing period" + float(value) + + +def test_the_selection_bias_warning_travels_with_the_numbers() -> None: + """The ledger states it at full strength and the README must not quietly drop it. + + Every configuration in the table is the argmax of a 144-cell slice selected on the same data + it is re-priced on. Quoting those as an asset's expected profit factor is exactly what the + experiment record forbids, and a benchmark that omitted the caveat would be a stronger claim + than the measurement supports -- in a block whose entire point is not doing that. + """ + block = rfr.render(_LEDGER_TEXT) + assert "never as edge estimates" in block + assert "argmax" in block + assert "zero *fee*, not zero cost" in block, ( + "slippage is held at 0.0005 in every cell, so the zero column is zero FEE and not zero " + "cost -- the ledger's own `validation` field says so and the block must too" + ) + + +@pytest.mark.parametrize( + ("mutation", "why"), + [ + ("PF by fee_pct 0/0.001", "no per-asset curve at all"), + ("BTC (n=123) 1.090/0.961", "asset curves with no fee-column list"), + ( + # The break-even clause is SEPARATE prose in the real row, and it has to be here + # too: without it the missing-break-even guard fires first and the count guard is + # never reached -- which is exactly how the first version of this case passed + # against a renderer with the count guard deleted. + "PF by fee_pct 0/0.001/0.002: BTC (n=123) 1.090/0.961. " + "Brackets: BTC 0.060%->1.0096 / 0.070%->0.9971 => 0.068%", + "two values against three columns -- the case that reaches the count guard", + ), + ( + "PF by fee_pct 0/0.012: BTC (n=123) 1.090/0.333", + "a curve with no measured break-even", + ), + ], +) +def test_the_renderer_refuses_a_ledger_row_it_cannot_fully_parse(mutation: str, why: str) -> None: + """Renders the ledger's numbers or it does not ship. A partial parse is the quiet + half-truth the block exists to refuse, so every recognition failure is fatal.""" + row = json.loads([line for line in _LEDGER_TEXT.splitlines() if rfr._SESSION in line][-1]) + row["params"]["fee_curve"] = mutation + with pytest.raises(SystemExit): + rfr.render(json.dumps(row)) + + +def test_the_renderer_refuses_a_readme_without_sentinels() -> None: + """It will not guess where the benchmark belongs.""" + with pytest.raises(SystemExit, match="sentinels"): + rfr.replace_block("# keel\n\nnothing here\n", "block")