From e3113ab9e7a093fcb2854e06cd97eac72a29dbf3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 09:27:24 -0800 Subject: [PATCH 1/2] docs(proposals): 0012 bounded trace ranking for drill-down Drill-down ranks every retained trace in its window, so it times out once the server holds data. Propose exactly maintained per-bucket rank lists in the read cache, and add the filebench scale benchmark that measures the current cost. --- docs/proposals/0012-bounded-trace-ranking.md | 230 ++++++++++++++++++ docs/proposals/README.md | 1 + .../trace_candidate_scale_bench_test.go | 106 ++++++++ 3 files changed, 337 insertions(+) create mode 100644 docs/proposals/0012-bounded-trace-ranking.md create mode 100644 internal/engine/trace_candidate_scale_bench_test.go diff --git a/docs/proposals/0012-bounded-trace-ranking.md b/docs/proposals/0012-bounded-trace-ranking.md new file mode 100644 index 00000000..fac5fb36 --- /dev/null +++ b/docs/proposals/0012-bounded-trace-ranking.md @@ -0,0 +1,230 @@ +# Bounded trace ranking for drill-down + +Status: Draft +Date: 2026-11-18 +Pull request: TBD + +## Summary + +Drill-down ranks every retained trace in its window to return the 21 slowest +or failing ones, so its cost grows with retained data and it times out on a +server that has run for a while. This proposal keeps a small, exactly +maintained ranked list of traces per time bucket and scope in the read cache. +A query merges the lists of the buckets inside its window and scans exactly +only what the lists cannot answer. Cost then depends on window length and list +size, not on how many traces are retained. + +## Motivation + +`queryrows.CompletedTraceQuery` serves service-map drill-down +(`indexedExemplars` in `internal/query/exemplars.go`, limit 21) and the +recent-trace lookup (`observability.Service.Trace`, limit 1). It orders every +in-window trace candidate by `has_error DESC, max_end-min_start DESC`. + +Evidence: + +- pitot-bench on a Hetzner ccx33 pair, `v2026.11.9-alpha`, 48k items/s mixed + ingest with 20 UI reads/s. A fresh server (about 15M rows) passes the read + gate at 20/20 ops/s on 3.25 cores. After a 10-minute prefill (about 180M + rows) the same load completes 2.6/20 ops/s on 7.3 cores. Every one of 1,716 + failures is a 504 on `/api/queries/exemplars`, and no drill-down or trace + detail succeeds. A local run on an M-series Mac reproduces it (4.8/20 ops/s). +- `internal/engine/trace_candidate_scale_bench_test.go` (build tag + `filebench`) times the query against a fully cached index with a two-batch + uncached tail: + + | Cached traces | Single-service traces | Two-service traces | + |---|---|---| + | 400k | 21 ms | 93 ms | + | 1.6M | 197 ms | 633 ms | + + Four times the traces costs seven to nine times the time. The Hetzner run + held roughly 30M traces; a day at 48k rows/s holds billions of rows, and + dashboard windows reach 24h. +- `EXPLAIN ANALYZE` at 1.6M two-service traces (0.59 s): every candidate has + `service=NULL`, so a service scope takes the repair path and aggregates every + retained `read_trace_parts` row (`HASH_JOIN RIGHT_SEMI` 207 ms with a 64,000 + row estimate against 1.6M actual, `HASH_GROUP_BY` 258 ms, a second 206 ms + group-by for the boundary subquery). DuckDB factors `trace_candidates` into a + materialized common subplan despite the `NOT MATERIALIZED` hint and spills + 275 MB. The top-N steps themselves cost about 4 ms. + +Rewriting the joins does not help once the index is fully cached (603 ms +against 624 ms). The work is proportional to retained traces by construction. + +## Goals and non-goals + +Goals: + +- Drill-down and recent-trace cost independent of retained traces, for every + scope (namespace and service set or empty) and window up to the retention + period. +- Exactly the current results, including ties (`trace_id`, then + `namespace`), late publication, compaction and retention. +- Bounded maintenance cost per cached batch. + +Non-goals: + +- Changing the ranking key or the definition of a trace's duration + (`max_end - min_start` over in-window, in-scope spans). +- Approximate ranking. Peers rank by root-span duration (SigNoz, Honeycomb, + Uptrace), return the first N matches (Tempo, Jaeger), or accept per-block + durations that understate split traces (Tempo). Pitot keeps its exact + definition. +- Trace search with arbitrary attribute filters, which keeps the generic path. + +## Design + +### Rank lists + +A new private read-cache table, `read_trace_rank`: + +| Column | Meaning | +|---|---| +| `tier` | minute, hour or day | +| `bucket` | bucket start, Unix nanoseconds | +| `namespace`, `service` | scope; `service` is `''` for the all-services list | +| `trace_id`, `min_start`, `max_start`, `max_end`, `has_error` | the trace's score in that scope | + +Each (tier, bucket, namespace, service) group holds the top +`queryrows.TraceRankDepth` (21) traces by the production ordering. A trace +belongs to the minute bucket of its `min_start`. Only short traces +(`max_start - min_start` under one minute) enter lists. + +A trace whose spans cover several services gets one row per service. Each +row's score is computed from that service's parts, because a service-scoped +query ranks only that service's spans. Rank rows store results only. The +"one incremental candidate per trace" rule still holds for +`read_trace_candidates`. + +### Why it is exact + +Each short trace lies in exactly one minute bucket per scope, so buckets +partition the traces. If a trace is in the global top K of a window, at most +K-1 traces outrank it anywhere, so at most K-1 outrank it in its own bucket. +It is therefore in its bucket's list, and the union of the lists of fully +covered buckets contains the global top K over those buckets. Hour and day +lists are the top K of their child lists, by the same argument. + +Scores come from merged candidates, never from per-batch parts. A newly +published span only lowers `min_start`, raises `max_end` or sets `has_error`, +so a trace's score never falls on publication. Updating a list is then an +insert with eviction of the lowest entry. Two events can lower a score or move +a trace between buckets: retention retiring part of a trace, and `min_start` +moving into an earlier bucket. Both rebuild the affected bucket from +`read_trace_candidates`, at a cost bounded by that bucket's traces. + +### Maintenance + +In the same `cacheBatches` transaction that inserts parts, merges candidates +and writes markers (`internal/engine/batch_cache.go`): + +1. Before the candidate MERGE, record the bucket keys of the candidates it + touches. After the MERGE, record their new keys. +2. Recompute those minute groups from `read_trace_candidates` and, for + per-service scores, from that bucket's `read_trace_parts`, using + `QUALIFY row_number() OVER (PARTITION BY namespace, service ORDER BY ...) + <= 21`. +3. Recompute affected hour groups from their minute lists and affected day + groups from their hour lists. + +Retention repair runs the same steps for the keys in `cache_retired_traces`. +A compaction covered by a cached replacement chain changes no candidate and +does no rank work. An uncovered one is treated as retention. Contributions, +rank rows and markers commit together, so a reader in one read transaction +under the pinned snapshot sees them consistently. + +### Query + +`CompletedTraceQuery` and its tests stay unchanged. `aggregateSources` and +`traceCandidateSource` bind `trace_candidates` as the union of: + +- rank rows of the day, hour and minute buckets covered by the window, + splitting it greedily into the largest covered buckets. A bucket is covered + when it starts at or after the window start and ends at least one minute + before the window end, because a short trace can extend up to a minute past + its bucket. Every short trace in a covered bucket is then fully inside the + window, as the index path requires (`min_start >= S AND max_start < E`); +- an exact candidate scan of the uncovered minutes at each edge; +- an exact scan of long traces (`max_start - min_start` of one minute or more) + inside the window, a set bounded by how many long traces exist; +- the existing partial-trace and uncached-tail sources. + +A limit above `TraceRankDepth`, or a bucket that query-time repair has marked +dirty, falls back to the exact candidate scan for those buckets. + +Rows read per request are about (day + hour + minute buckets) × groups × 21, +plus the edge minutes, the long traces and the tail. For a 24h window that is +at most about 2 + 46 + 118 buckets, independent of retention. + +### Remaining scan + +The `touched` stage still needs the cached rows of traces that also appear in +the uncached tail, and the boundary `partial` filter still reads the index. +Neither has an index today: a per-trace index ran out of memory, and +`views_test.go` asserts that the read tables have none. The first build +bounds both by pruning `read_trace_candidates` on `min_start` within the tail +and edge time ranges. Phase 2 measures whether that is enough. + +### Versioning + +The rank table changes read-cache schema and semantics, so `readCacheVersion` +goes from 19 to 20, and a mismatch rebuilds every private read table. + +## Phases + +1. **Harness, done.** The scale benchmark above, with full cache refresh and + a two-service toggle. The pitot-bench `--prefill` phase + (labstack/pitot-bench#14) runs measured reads on a populated server. +2. **Minute tier.** `read_trace_rank` with minute buckets only, maintenance + in `cacheBatches` and retention repair, and query binding. Gate: the + existing exactness suite (`trace_candidate_window_test.go`) passes + unchanged, and an added oracle test compares random windows, scopes and + limits against raw spans after late publication, compaction and retention. + The scale benchmark must show drill-down time growing no more than the + number of minutes in the window between 1.6M, 16M and 30M traces. +3. **Hour and day tiers.** Gate: a 24h window costs under 50 ms at 30M + traces in the scale benchmark, and refresh cost per batch rises by no more + than 50%. +4. **End-to-end gate.** pitot-bench on Hetzner with a full-day prefill (about + 4B rows; about 4 hours of prefill on a ccx33 pair), then 48k items/s with + 20 reads/s. The read gate must pass. A 10-minute prefill is not enough, + because it holds about 1/20 of a day's data. + +## Alternatives + +- **Rewriting the joins.** No gain once the index is fully cached. +- **Per-batch top-K.** Inexact, because a trace split across batches has a + merged duration that no batch's partial shows. Its cost also grows with the + number of batches. +- **Threshold algorithms (Fagin TA/NRA) over per-batch partials.** Unseen + batches can extend any trace, so the only available upper bounds are + window-wide and too loose to stop early. +- **Ordered index on the score.** Exact, but DuckDB indexes do not serve + ordered limit scans, and a per-trace index already ran out of memory. +- **Sampling, or Jaeger-style per-bucket `LIMIT N*k`.** Approximate, with no + bound on missing the true top K. + +## Risks + +- Maintenance cost. Measured on a simulated table: 7–12 ms per 40k-part + batch, against about 25 ms per batch for today's refresh. Recomputing only + touched buckets should be cheaper; phase 2 measures it. +- Long traces bypass the lists. A workload of mostly long traces degrades + toward today's cost. Phase 2 reports their share in the benchmark and on + Hetzner. +- Planner estimates. Bad cardinality estimates already reversed a hash build + in the exemplar path. Bound the new sources with explicit predicates and + check the plans in tests. +- Materialization. DuckDB 2 may factor repeated sources into materialized + subplans regardless of hints. Keep the merged rank source small enough that + materializing it is cheap. + +## Open questions + +- Should the one-minute short-trace threshold follow the bucket width or be + measured from real span-duration distributions? +- Is zonemap pruning on `min_start` enough for the tail join, or does the + tail need its own bounded structure? +- Rank depth: 21 fits both callers. Should the trace search page reuse the + lists at a larger depth? diff --git a/docs/proposals/README.md b/docs/proposals/README.md index 385fac3f..5acc6020 100644 --- a/docs/proposals/README.md +++ b/docs/proposals/README.md @@ -38,5 +38,6 @@ Earlier proposals predate the template and keep their original structure. | 0004 | [Full UI without an LLM key, and PitotQL](0004-pitotql.md) | Accepted | | 0005 | [Dedicated attribute columns and batch format 5](0005-dedicated-attribute-columns.md) | Accepted | | 0006 | [Read performance under sustained ingest](0006-read-performance.md) | Draft | +| 0012 | [Bounded trace ranking for drill-down](0012-bounded-trace-ranking.md) | Draft | Settled decisions that a proposal leads to are recorded in [../adr](../adr/README.md). diff --git a/internal/engine/trace_candidate_scale_bench_test.go b/internal/engine/trace_candidate_scale_bench_test.go new file mode 100644 index 00000000..5c8b7276 --- /dev/null +++ b/internal/engine/trace_candidate_scale_bench_test.go @@ -0,0 +1,106 @@ +//go:build filebench + +package engine + +import ( + "fmt" + "os" + "sort" + "strconv" + "testing" + "time" + + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" + telemetrystore "github.com/labstack/pitot/internal/telemetry/store" +) + +// Time the completed-trace kernel as retained traces grow, with a raw tail of +// two uncached recent batches like live ingest. PITOT_SCALE_BATCHES sets the +// fully cached batches of 20,000 two-span traces each; PITOT_SCALE_MIXED=1 puts +// each trace's spans in two services, as real request traces do. +func TestCompletedTraceQueryScaleBenchmark(t *testing.T) { + mixed := os.Getenv("PITOT_SCALE_MIXED") == "1" + batches := 20 + if v, err := strconv.Atoi(os.Getenv("PITOT_SCALE_BATCHES")); err == nil && v > 0 { + batches = v + } + const traces = 20000 + d, repo := newBatchCacheTest(t) + base := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) + commit := func(part int) { + batch := telemetrystore.Batch{ID: fmt.Sprintf("part_%d", part)} + for i := range traces { + id := uint64(part*traces+i+1) * 0x9e3779b97f4a7c15 + start := base.Add(time.Duration(part)*time.Second + time.Duration(i%1000)*time.Millisecond) + status := "STATUS_CODE_OK" + if i%97 == 0 { + status = "STATUS_CODE_ERROR" + } + for span := range 2 { + service := "api" + if mixed && span == 1 { + service = "db" + } + batch.Spans = append(batch.Spans, telemetry.Span{Namespace: "prod", ServiceName: service, + TraceID: fmt.Sprintf("%032x", id), SpanID: fmt.Sprintf("%016x", span+1), + StartUnixNanos: start.UnixNano(), EndUnixNanos: start.Add(time.Duration(1+i%50) * time.Millisecond).UnixNano(), + DurationMS: float64(1 + i%50), StatusCode: status, IngestedAt: start.UnixNano()}) + } + } + if err := repo.Commit(t.Context(), batch); err != nil { + t.Fatal(err) + } + } + for part := range batches { + commit(part) + } + // One pass stops at the row budget; repeat until every batch is cached. + for { + rows, err := d.RefreshReadCaches(t.Context()) + if err != nil { + t.Fatal(err) + } + if rows == 0 { + break + } + } + // Recent batches the read cache has not reached yet. + for part := batches; part < batches+2; part++ { + commit(part) + } + w := queryrows.Window{Start: base.Add(-time.Hour), End: base.Add(2 * time.Hour), Namespace: "prod", Service: "api", Kind: queryrows.TraceCandidateRead} + args := []any{w.Start, w.End, w.Namespace, w.Namespace, w.Service, w.Service, 21} + run := func(query string) []string { + rows, err := d.QueryContext(queryrows.WithWindow(t.Context(), w), query, args...) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + var out []string + for rows.Next() { + var namespace, trace string + var hasError int + if err := rows.Scan(&namespace, &trace, &hasError); err != nil { + t.Fatal(err) + } + out = append(out, fmt.Sprint(namespace, trace, hasError)) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + return out + } + if got := run(queryrows.CompletedTraceQuery); len(got) != 21 { + t.Fatalf("completed rows=%d", len(got)) + } + const samples = 15 + var ms []float64 + for range samples { + started := time.Now() + run(queryrows.CompletedTraceQuery) + ms = append(ms, float64(time.Since(started))/float64(time.Millisecond)) + } + sort.Float64s(ms) + t.Logf("cached_traces=%d tail_traces=%d mixed_services=%v median_ms=%.1f max_ms=%.1f", batches*traces, 2*traces, mixed, ms[samples/2], ms[samples-1]) +} From 5c5cd41561ad8f45b10d80f08746db5851f8b23c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 09:27:41 -0800 Subject: [PATCH 2/2] docs(proposals): link 0012 to its pull request --- docs/proposals/0012-bounded-trace-ranking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposals/0012-bounded-trace-ranking.md b/docs/proposals/0012-bounded-trace-ranking.md index fac5fb36..af804cb7 100644 --- a/docs/proposals/0012-bounded-trace-ranking.md +++ b/docs/proposals/0012-bounded-trace-ranking.md @@ -2,7 +2,7 @@ Status: Draft Date: 2026-11-18 -Pull request: TBD +Pull request: #504 ## Summary