diff --git a/README.md b/README.md index 1ff1fd04..170b4fe8 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,39 @@ The point of this project is the enforcement machinery and the honest measuremen runs through it, not a claim of alpha. A visitor who finds that out themselves feels misled; one who is told upfront can read it as rigour. + + +### The benchmark: the same rule, priced twice + +`turtle_breakout` on hourly bars, at each asset's best-swept configuration. The left column is +the number a fee-blind backtest reports. The right is the same run priced at the taker +fee this account actually pays. Nothing else changes between them. + +| asset | trades | PF at 0% fee | PF at 1.2% taker | break-even fee, measured | +| :-- | ---: | ---: | ---: | ---: | +| BTC | 123 | 1.090 | 0.333 | 0.068% | +| ETH | 121 | 1.458 | 0.556 | 0.433% | +| SOL | 92 | 1.533 | 0.801 | 0.751% | +| ZEC | 50 | 2.713 | 1.303 | 1.741% | + +Four of four are profitable with the fee removed. **Zero of four survive the fee that is +actually charged**, and the break-even rate varies by a factor of ~26 across assets +running the same rule on the same clock over the same window — the asset is a far larger +lever than any parameter in an 864-trial sweep. + +**Read these as a comparison, never as edge estimates.** Every configuration above is the +argmax of that asset's 144-cell slice, selected on the same data it is re-priced on — a +maximum of 144 draws, not an expectation. The bias runs *against* the finding, which is +why the comparison survives it: it inflates the arm that wins with the fee removed, and +that arm still dies when the fee is charged. Break-even fees were bracketed by real +cells,not interpolated. Slippage is held at 0.0005 in every cell, so the zero column is +zero *fee*, not zero cost. + +Source: [`docs/experiments/2026-08-12-fee-curve-and-rsi-meanrev.md`](docs/experiments/2026-08-12-fee-curve-and-rsi-meanrev.md), rendered from the hash-chained trials ledger by +`scripts/render_fee_reality.py`. + + + **The cadence problem, and the pipeline built to solve it:** the promotion gate's 100-trade floor is honest only if the sample is collectable — and at the daily clock's measured 2.15 signals per asset-year, it is 31–84 years away per asset. Waiting is not a slower diff --git a/scripts/render_fee_reality.py b/scripts/render_fee_reality.py new file mode 100644 index 00000000..5599af1d --- /dev/null +++ b/scripts/render_fee_reality.py @@ -0,0 +1,206 @@ +"""Render the fee-reality benchmark block from the trials ledger (#646). + +**Nothing here computes a result.** Every number is read out of +`docs/experiments/trials-ledger.jsonl` -- the hash-chained record of what was actually run -- +and reformatted. That is the issue's binding constraint: the block renders the ledger's numbers +or it does not ship, because a hand-typed benchmark is a marketing claim wearing a measurement's +clothes, and this project's whole argument is the difference between those two. + +Run it to regenerate the README block in place: + + python scripts/render_fee_reality.py --write + +`tests/test_fee_reality_block.py` fails when the file and this script's output disagree, so the +README cannot drift away from the measurement without the suite saying so. + +**Parsing prose is deliberate, and its failure mode is the point.** The ledger stores the fee +curve as a recorded sentence rather than a table, because that row was written by the experiment +that produced it and rewriting history to suit a renderer would be the tail wagging the dog. So +this parses it -- strictly, with every pattern anchored, raising on anything it does not +recognise. A parse that silently produced three of four assets would be exactly the quiet +half-truth the block exists to refuse. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass +from decimal import Decimal +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +_LEDGER = _ROOT / "docs/experiments/trials-ledger.jsonl" +_README = _ROOT / "README.md" + +#: The ledger session that re-priced already-ledgered configurations at additional fee rates. +_SESSION = "fee-curve-and-rsi-meanrev-2026-08-12" + +BEGIN = "" +END = "" + +#: The taker rate this account actually pays, as the fee curve's own last column. The ledger +#: writes `fee_pct` as a fraction of notional; `_as_percent` is what turns it into the number an +#: operator recognises from a fee schedule. +_TAKER_COLUMN = "0.012" + + +def _as_percent(fee_pct: str) -> str: + """`"0.012"` -> `"1.2%"`. A fee schedule quotes percentages, and a README that quoted + fractions would make the reader do the conversion that produced #247's costing error.""" + value = Decimal(fee_pct) * 100 + return f"{value.normalize():f}%" + + +@dataclass(frozen=True) +class AssetCurve: + """One asset's measured profit factor across the fee sweep.""" + + asset: str + trades: int + #: fee_pct (as written in the ledger) -> profit factor, in the ledger's own column order. + by_fee: dict[str, str] + #: The MEASURED break-even fee, bracketed by real cells rather than interpolated. + break_even_pct: str + + +def _row(ledger_text: str) -> dict: + """The one ledger row carrying the fee curve, or a loud failure.""" + rows = [json.loads(line) for line in ledger_text.splitlines() if line.strip()] + curves = [ + r + for r in rows + if r.get("session") == _SESSION and "fee_curve" in r.get("params", {}) + ] + if len(curves) != 1: + raise SystemExit( + f"expected exactly one {_SESSION!r} row carrying a fee_curve, found {len(curves)}. " + "The ledger is append-only and hash-chained, so this means the renderer's " + "expectation is stale -- not that the record is wrong." + ) + return curves[0] + + +def parse(ledger_text: str) -> tuple[list[AssetCurve], list[str], str]: + """`(curves, fee columns, the rule the curve was measured on)` from the ledger's own prose.""" + params = _row(ledger_text)["params"] + curve = params["fee_curve"] + + columns = re.search(r"PF by fee_pct ([\d./]+):", curve) + if columns is None: + raise SystemExit("could not find the fee-column list in the ledger's fee_curve text") + fees = columns.group(1).split("/") + + # `BTC (n=123) 1.090/0.961/...` -- anchored on the parenthesised trade count so a sentence + # elsewhere in the row cannot match by accident. + # `\d+\.\d+`, never `[\d.]+`: the latter is greedy enough to swallow the SENTENCE'S full + # stop into the last asset's profit factor, which is how the first draft rendered ZEC as + # "1.303." -- a number that is still readable, still wrong, and would have shipped. + entries = re.findall(r"\b([A-Z]{2,10}) \(n=(\d+)\) (\d+\.\d+(?:/\d+\.\d+)+)", curve) + if not entries: + raise SystemExit("could not find any per-asset fee curve in the ledger's fee_curve text") + + # `=> 0.068%` closes each measured bracket; the rsi_meanrev one names its rule and is skipped. + breakevens = dict( + re.findall(r"(?/\s-]*?=> ([\d.]+)%", curve) + ) + + curves: list[AssetCurve] = [] + for asset, trades, factors in entries: + values = factors.split("/") + if len(values) != len(fees): + raise SystemExit( + f"{asset}: {len(values)} profit factors against {len(fees)} fee columns -- " + "refusing to render a row whose columns do not line up" + ) + if asset not in breakevens: + raise SystemExit(f"{asset}: no measured break-even fee in the ledger row") + curves.append( + AssetCurve( + asset=asset, + trades=int(trades), + by_fee=dict(zip(fees, values)), + break_even_pct=breakevens[asset], + ) + ) + return curves, fees, params["fee_curve_rule"].split()[0] + + +def render(ledger_text: str) -> str: + """The README block. Pure: same ledger in, same markdown out.""" + curves, fees, rule = parse(ledger_text) + document = _row(ledger_text)["params"]["document"] + zero = fees[0] + + lines = [ + BEGIN, + "", + "### The benchmark: the same rule, priced twice", + "", + f"`{rule}` on hourly bars, at each asset's best-swept configuration. The left column is", + "the number a fee-blind backtest reports. The right is the same run priced at the taker", + "fee this account actually pays. Nothing else changes between them.", + "", + f"| asset | trades | PF at {_as_percent(zero)} fee | PF at {_as_percent(_TAKER_COLUMN)} " + "taker | break-even fee, measured |", + "| :-- | ---: | ---: | ---: | ---: |", + ] + for c in curves: + lines.append( + f"| {c.asset} | {c.trades} | {c.by_fee[zero]} | {c.by_fee[_TAKER_COLUMN]} | " + f"{c.break_even_pct}% |" + ) + lines += [ + "", + "Four of four are profitable with the fee removed. **Zero of four survive the fee that is", + "actually charged**, and the break-even rate varies by a factor of ~26 across assets", + "running the same rule on the same clock over the same window — the asset is a far larger", + "lever than any parameter in an 864-trial sweep.", + "", + "**Read these as a comparison, never as edge estimates.** Every configuration above is the", + "argmax of that asset's 144-cell slice, selected on the same data it is re-priced on — a", + "maximum of 144 draws, not an expectation. The bias runs *against* the finding, which is", + "why the comparison survives it: it inflates the arm that wins with the fee removed, and", + "that arm still dies when the fee is charged. Break-even fees were bracketed by real", + "cells," + "not interpolated. Slippage is held at 0.0005 in every cell, so the zero column is", + "zero *fee*, not zero cost.", + "", + f"Source: [`{document}`]({document}), rendered from the hash-chained trials ledger by", + "`scripts/render_fee_reality.py`.", + "", + END, + ] + return "\n".join(lines) + + +def replace_block(readme: str, block: str) -> str: + """`readme` with the sentinel block swapped for `block`.""" + start, end = readme.find(BEGIN), readme.find(END) + if start == -1 or end == -1 or end < start: + raise SystemExit( + f"README.md is missing the {BEGIN!r} / {END!r} sentinels -- refusing to guess where " + "the benchmark belongs" + ) + return readme[:start] + block + readme[end + len(END) :] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--write", action="store_true", help="rewrite README.md in place") + args = parser.parse_args(argv) + block = render(_LEDGER.read_text(encoding="utf-8")) + if not args.write: + print(block) + return 0 + _README.write_text( + replace_block(_README.read_text(encoding="utf-8"), block), encoding="utf-8" + ) + print(f"wrote the fee-reality block into {_README}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_fee_reality_block.py b/tests/test_fee_reality_block.py new file mode 100644 index 00000000..55f89be3 --- /dev/null +++ b/tests/test_fee_reality_block.py @@ -0,0 +1,136 @@ +"""The README's fee-reality benchmark is RENDERED from the ledger, never typed (#646). + +The block is the project's opening claim -- "no shipped rule family is net-positive at the taker +fee actually paid" -- shown as numbers rather than asserted as prose. Its whole persuasive force +is that it comes from the hash-chained record of what was actually run, so a figure edited by +hand would not be a small inaccuracy: it would be the one claim this repository makes about +itself, made the way it says nobody should. + +These tests are the mechanism. The README and `scripts/render_fee_reality.py` must agree +byte-for-byte, and the renderer must refuse to emit anything at all from a ledger row it does +not fully recognise. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_ROOT / "scripts")) + +import render_fee_reality as rfr # noqa: E402 + +_LEDGER_TEXT = (_ROOT / "docs/experiments/trials-ledger.jsonl").read_text(encoding="utf-8") +_README = (_ROOT / "README.md").read_text(encoding="utf-8") + + +def test_the_readme_block_matches_what_the_ledger_renders() -> None: + """The drift pin. Edit the README's numbers and this fails; re-run the renderer and it + passes -- which is the only way the block is allowed to change.""" + block = rfr.render(_LEDGER_TEXT) + assert block in _README, ( + "README.md's fee-reality block is not what scripts/render_fee_reality.py produces from " + "the ledger. Regenerate it (`python scripts/render_fee_reality.py --write`) rather than " + "editing the numbers -- a hand-edited benchmark is the failure this pin exists for." + ) + + +def test_the_block_leads_the_readme_before_the_feature_list() -> None: + """#646's first acceptance line: the benchmark comes before what keel can do. + + A visitor who reads the capabilities first and the result second has been sold to and then + corrected. The order is the honesty. + """ + block_at = _README.find(rfr.BEGIN) + assert block_at != -1, "the fee-reality block is not in README.md at all" + for later in ("## ", "### What it does", "### Install"): + found = _README.find(later) + if found != -1: + assert block_at < found, f"the benchmark sits after {later!r}" + + +def test_every_number_in_the_block_appears_in_the_ledger_row() -> None: + """No figure may exist in the README that is not in the record. + + Stronger than "the renderer produced it", because a renderer with a literal baked in would + also satisfy the drift pin -- both sides would simply carry the same invention. + """ + curves, fees, _rule = rfr.parse(_LEDGER_TEXT) + row = rfr._row(_LEDGER_TEXT)["params"]["fee_curve"] + assert curves, "no asset curves parsed -- this test would prove nothing" + for curve in curves: + assert f"n={curve.trades}" in row + for value in curve.by_fee.values(): + assert value in row, f"{curve.asset}: {value} is not in the ledger row" + assert f"{curve.break_even_pct}%" in row + + +def test_a_profit_factor_never_carries_a_stray_sentence_period() -> None: + """The bug the first draft shipped into its own output. + + `[\\d.]+` is greedy enough to swallow the sentence's full stop into the LAST asset's profit + factor, and "1.303." is still readable, still wrong, and would have gone unnoticed in a + table. Anchored on the last asset specifically, because that is the only position where the + sentence can reach. + """ + curves, _fees, _rule = rfr.parse(_LEDGER_TEXT) + for curve in curves: + for value in curve.by_fee.values(): + assert not value.endswith("."), f"{curve.asset}: {value!r} has a trailing period" + float(value) + + +def test_the_selection_bias_warning_travels_with_the_numbers() -> None: + """The ledger states it at full strength and the README must not quietly drop it. + + Every configuration in the table is the argmax of a 144-cell slice selected on the same data + it is re-priced on. Quoting those as an asset's expected profit factor is exactly what the + experiment record forbids, and a benchmark that omitted the caveat would be a stronger claim + than the measurement supports -- in a block whose entire point is not doing that. + """ + block = rfr.render(_LEDGER_TEXT) + assert "never as edge estimates" in block + assert "argmax" in block + assert "zero *fee*, not zero cost" in block, ( + "slippage is held at 0.0005 in every cell, so the zero column is zero FEE and not zero " + "cost -- the ledger's own `validation` field says so and the block must too" + ) + + +@pytest.mark.parametrize( + ("mutation", "why"), + [ + ("PF by fee_pct 0/0.001", "no per-asset curve at all"), + ("BTC (n=123) 1.090/0.961", "asset curves with no fee-column list"), + ( + # The break-even clause is SEPARATE prose in the real row, and it has to be here + # too: without it the missing-break-even guard fires first and the count guard is + # never reached -- which is exactly how the first version of this case passed + # against a renderer with the count guard deleted. + "PF by fee_pct 0/0.001/0.002: BTC (n=123) 1.090/0.961. " + "Brackets: BTC 0.060%->1.0096 / 0.070%->0.9971 => 0.068%", + "two values against three columns -- the case that reaches the count guard", + ), + ( + "PF by fee_pct 0/0.012: BTC (n=123) 1.090/0.333", + "a curve with no measured break-even", + ), + ], +) +def test_the_renderer_refuses_a_ledger_row_it_cannot_fully_parse(mutation: str, why: str) -> None: + """Renders the ledger's numbers or it does not ship. A partial parse is the quiet + half-truth the block exists to refuse, so every recognition failure is fatal.""" + row = json.loads([line for line in _LEDGER_TEXT.splitlines() if rfr._SESSION in line][-1]) + row["params"]["fee_curve"] = mutation + with pytest.raises(SystemExit): + rfr.render(json.dumps(row)) + + +def test_the_renderer_refuses_a_readme_without_sentinels() -> None: + """It will not guess where the benchmark belongs.""" + with pytest.raises(SystemExit, match="sentinels"): + rfr.replace_block("# keel\n\nnothing here\n", "block")