From 5c956dfc8fec4a7a7a24b9a7b531e71125cd9741 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 02:36:10 -0800 Subject: [PATCH 1/2] perf(query): admit the observability and annotation readers The observability readers (overview, topology, dependencies, trace, health_trend) and the annotation reader call the engine directly, bypassing the executor's read admission. Under a saturating dashboard they oversubscribe the DuckDB read pool, block on DB.Conn, and defeat the deadline shedding the pool size is meant to enforce, so reads collapse under concurrent ingest (issue #480). Route those readers through the existing interactive admission lane with an admittingQueryer wrapper. The wrapper holds the interactive slot across row iteration and releases it on Close, bounds the wait by the batch timeout, and forwards DefaultNamespace and ReadTransactioner so annotations keep their single-snapshot read. Excess load now sheds as a bounded ErrBusy (503 with Retry-After) instead of piling up inside the engine. The panel-internal rollup reader stays on the raw engine: its reads already run inside an admitted panel slot, so admitting them again would deadlock a one-slot pool. This is Phase 0 of proposal 0011. It makes read load degrade gracefully; it does not by itself meet the sustained read-rate gate, which depends on the read-cost work in proposals 0007 and 0010. --- cmd/pitot/main.go | 12 ++- internal/api/annotations.go | 7 +- internal/api/annotations_test.go | 10 +- internal/config/sizing.go | 6 +- internal/query/admit_reader.go | 114 ++++++++++++++++++++ internal/query/admit_reader_test.go | 155 ++++++++++++++++++++++++++++ 6 files changed, 295 insertions(+), 9 deletions(-) create mode 100644 internal/query/admit_reader.go create mode 100644 internal/query/admit_reader_test.go diff --git a/cmd/pitot/main.go b/cmd/pitot/main.go index 63786a91..5a2eb983 100644 --- a/cmd/pitot/main.go +++ b/cmd/pitot/main.go @@ -298,8 +298,11 @@ func main() { // typed query kernel through deterministic HTTP or standard MCP tools. // Route both HTTP and MCP reads through Duck's retrying adapter. Passing the // raw *sql.DB here bypassed the Telemetry maintenance-race protection. - queries := observability.New(q, q, cfg.RetentionDays) panels := query.NewExecutor(q, cfg.RetentionDays) + // The observability readers call the engine directly, so route them through + // the executor's interactive admission lane; otherwise they oversubscribe the + // read pool and starve admitted panels under a saturating dashboard. + queries := observability.New(panels.AdmitReader(q), q, cfg.RetentionDays) writer.SetAdmission(repository.CompactionDebt, panels.InteractiveRefusals) go func() { err := writer.Run(ctx) @@ -313,11 +316,14 @@ func main() { }() q.SetReadPressure(panels.Pressure) go q.RunRollups(ctx) - panels.SetRollupReader(queries) + // Panel rollup reads already run inside an admitted panel slot, so the rollup + // reader uses the raw engine. Admitting it again would make a one-slot pool + // deadlock: the panel holds the only slot while its nested read waits for one. + panels.SetRollupReader(observability.New(q, q, cfg.RetentionDays)) panels.SetCacheTTL(cfg.QueryCacheTTL) api.RegisterQueryRoutes(e, panels) api.RegisterPitotQLRoutes(e, q, panels) - api.RegisterAnnotationRoutes(e, annotations.New(q)) + api.RegisterAnnotationRoutes(e, annotations.New(panels.AdmitReader(q))) api.RegisterTraceRoutes(e, panels.CachedTraces(queries), cfg.RetentionDays) api.RegisterIntelligenceRoutes(e, detector) dashboards := dashboard.New(sqlite.DB, panels) diff --git a/internal/api/annotations.go b/internal/api/annotations.go index 9b5c2657..0087b586 100644 --- a/internal/api/annotations.go +++ b/internal/api/annotations.go @@ -7,6 +7,7 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/pitot/internal/annotations" + "github.com/labstack/pitot/internal/query" ) type AnnotationReader interface { @@ -21,16 +22,18 @@ func RegisterAnnotationRoutes(e *echo.Echo, service AnnotationReader) { } out, err := service.Read(c.Request().Context(), req) if err != nil { - return annotationError(err) + return annotationError(c, err) } return c.JSON(http.StatusOK, out) }, RequireCapability(ReadTelemetry)) } -func annotationError(err error) error { +func annotationError(c *echo.Context, err error) error { switch { case errors.Is(err, annotations.ErrRequest): return echo.NewHTTPError(http.StatusBadRequest, err.Error()).Wrap(err) + case errors.Is(err, query.ErrBusy): + return busyError(c) case errors.Is(err, context.DeadlineExceeded): return echo.NewHTTPError(http.StatusGatewayTimeout, "Annotations took longer than 10 seconds").Wrap(err) case errors.Is(err, context.Canceled): diff --git a/internal/api/annotations_test.go b/internal/api/annotations_test.go index fb645c12..9054c815 100644 --- a/internal/api/annotations_test.go +++ b/internal/api/annotations_test.go @@ -12,6 +12,7 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/pitot/internal/annotations" + "github.com/labstack/pitot/internal/query" ) type annotationFake struct{} @@ -21,16 +22,20 @@ func (annotationFake) Read(context.Context, annotations.Request) (annotations.Re } func TestAnnotationErrorsWrapped(t *testing.T) { + e := echo.New() + newCtx := func() *echo.Context { + return e.NewContext(httptest.NewRequest(http.MethodPost, "/", nil), httptest.NewRecorder()) + } original := errors.New("storage failed") for _, cause := range []error{annotations.ErrRequest, context.DeadlineExceeded, original} { wrapped := fmt.Errorf("annotation query: %w", cause) - got := annotationError(wrapped) + got := annotationError(newCtx(), wrapped) var httpErr *echo.HTTPError if !errors.As(got, &httpErr) || !errors.Is(got, cause) { t.Fatalf("cause lost: %v", got) } } - if got := annotationError(fmt.Errorf("read: %w", context.Canceled)); got != nil { + if got := annotationError(newCtx(), fmt.Errorf("read: %w", context.Canceled)); got != nil { t.Fatalf("cancellation: %v", got) } } @@ -50,6 +55,7 @@ func TestAnnotationErrorMappings(t *testing.T) { {"request", annotations.ErrRequest, 400, annotations.ErrRequest.Error()}, {"deadline", context.DeadlineExceeded, 504, "Annotations took longer than 10 seconds"}, {"canceled", context.Canceled, 200, ""}, + {"busy", query.ErrBusy, 503, query.BusyMessage}, {"unknown", errors.New("storage failed"), 500, "annotations unavailable"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/config/sizing.go b/internal/config/sizing.go index d42a541f..f7b03dc6 100644 --- a/internal/config/sizing.go +++ b/internal/config/sizing.go @@ -51,8 +51,10 @@ const ( // QueryAdmission derives the executor's read admission from the pool: // slots reads hold connections at once. reserved slots serve interactive // reads only; queue bounds dashboard batch tickets (interactive waits are 2*slots). -// A one-slot pool has no reservation so panels can still execute. A pool of four or more keeps two connections -// for annotations and readiness checks, which do not queue behind them. +// A one-slot pool has no reservation so panels can still execute. A pool of four +// or more keeps two connections outside the slots for readiness checks and other +// direct engine reads (such as the panel-internal rollup reader) that must not +// queue behind admitted panels. func QueryAdmission(pool int) (slots, reserved, queue int) { slots = max(1, pool) if pool >= 4 { diff --git a/internal/query/admit_reader.go b/internal/query/admit_reader.go new file mode 100644 index 00000000..160b8d3d --- /dev/null +++ b/internal/query/admit_reader.go @@ -0,0 +1,114 @@ +package query + +import ( + "context" + "errors" + "sync" + + "github.com/labstack/pitot/internal/queryrows" +) + +// AdmitReader wraps a read surface so every read it issues passes through the +// interactive admission lane. The observability readers, annotations, and +// variable-option queries otherwise call the engine directly and compete for +// the read pool without being admitted, so a saturating dashboard oversubscribes +// the pool, starves admitted panels, and defeats the deadline shedding the pool +// size is meant to enforce. Routing those reads through admission restores the +// invariant that no more reads run concurrently than the pool can serve, and +// turns excess load into a bounded ErrBusy shed rather than unbounded queueing +// inside the engine. +// +// The admission slot is held for the lifetime of the returned rows and released +// when they close, mirroring how the engine holds a pool connection until its +// rows close. Callers already close every result set before issuing the next +// read, so one interactive slot is never held across acquiring another. +func (e *Executor) AdmitReader(inner queryrows.Queryer) queryrows.Queryer { + return &admittingQueryer{inner: inner, exec: e} +} + +type admittingQueryer struct { + inner queryrows.Queryer + exec *Executor +} + +func (a *admittingQueryer) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { + release, err := a.exec.admitInteractive(ctx) + if err != nil { + return nil, err + } + rows, err := a.inner.QueryContext(ctx, query, args...) + if err != nil { + release() + return nil, err + } + return &admittedRows{Rows: rows, release: release}, nil +} + +// WithReadTransaction admits the whole callback as one unit and runs it on the +// wrapped surface's own read transaction, so a reader that needs a single +// consistent snapshot (annotations reads deploys, anomalies, and the watermark +// together) keeps it and takes one slot for the run rather than one per +// sub-read. A surface without transaction support runs the callback against +// this admitting queryer instead, so its reads are still admitted. +func (a *admittingQueryer) WithReadTransaction(ctx context.Context, read func(queryrows.Queryer) error) error { + rt, ok := a.inner.(queryrows.ReadTransactioner) + if !ok { + return read(a) + } + release, err := a.exec.admitInteractive(ctx) + if err != nil { + return err + } + defer release() + return rt.WithReadTransaction(ctx, read) +} + +// DefaultNamespace forwards to the wrapped surface when it exposes one, matching +// the optional-interface probe callers use, and is empty otherwise. +func (a *admittingQueryer) DefaultNamespace() string { + if d, ok := a.inner.(interface{ DefaultNamespace() string }); ok { + return d.DefaultNamespace() + } + return "" +} + +// admitInteractive takes one interactive slot, bounding the wait by the batch +// timeout as the exemplar and trace-detail path does, so a deadline-less caller +// (a background or MCP read) cannot sit in the bounded queue indefinitely. A +// full queue or an expired wait is counted as an interactive refusal and +// returned as ErrBusy, except when the caller's own context ended first. These +// refusals share the executor's interactive-pressure counter, so a flood of +// routed observability or annotation reads also feeds ingest backpressure — +// intended, since interactive read starvation is exactly when ingest should +// yield. +func (e *Executor) admitInteractive(ctx context.Context) (func(), error) { + wait, cancel := context.WithTimeout(ctx, e.batchTimeout) + defer cancel() + err := e.acquireClass(wait, interactiveClass) + if errors.Is(err, errQueueFull) || errors.Is(err, context.DeadlineExceeded) { + e.interactiveRefusals.Add(1) + } + switch { + case err == nil: + return func() { e.admission.release(interactiveClass) }, nil + case ctx.Err() != nil: + return nil, ctx.Err() + default: + return nil, ErrBusy + } +} + +// admittedRows releases the admission slot when the wrapped rows close. Close is +// idempotent, so a double close or a close after exhaustion releases the slot +// exactly once. +type admittedRows struct { + queryrows.Rows + release func() + once sync.Once +} + +func (r *admittedRows) Close() error { + err := r.Rows.Close() + r.once.Do(r.release) + return err +} diff --git a/internal/query/admit_reader_test.go b/internal/query/admit_reader_test.go new file mode 100644 index 00000000..116ba790 --- /dev/null +++ b/internal/query/admit_reader_test.go @@ -0,0 +1,155 @@ +package query + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/labstack/pitot/internal/queryrows" +) + +type fakeRows struct{ closed *bool } + +func (fakeRows) Columns() ([]string, error) { return nil, nil } +func (fakeRows) Err() error { return nil } +func (fakeRows) Next() bool { return false } +func (fakeRows) Scan(...any) error { return nil } +func (r fakeRows) Close() error { *r.closed = true; return nil } + +type fakeReader struct { + namespace string + rows queryrows.Rows + queried *bool + txRan *bool +} + +func (f fakeReader) QueryContext(context.Context, string, ...any) (queryrows.Rows, error) { + if f.queried != nil { + *f.queried = true + } + return f.rows, nil +} + +func (f fakeReader) DefaultNamespace() string { return f.namespace } + +func (f fakeReader) WithReadTransaction(ctx context.Context, read func(queryrows.Queryer) error) error { + if f.txRan != nil { + *f.txRan = true + } + return read(f) +} + +func newAdmissionFixture(t *testing.T, slots, reserved, queue int) *Executor { + t.Helper() + e := NewExecutor(pooledEngine{pool: 8}, 30) + t.Cleanup(e.Close) + e.admission = newReadAdmission(slots, reserved, queue) + return e +} + +// A routed read holds its admission slot until the rows close, then frees it, so +// a bypass reader can no longer occupy the pool after it has finished reading. +func TestAdmittedReaderHoldsItsSlotUntilRowsClose(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 4) + closed := false + reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: &closed}}) + rows, err := reader.QueryContext(t.Context(), "SELECT 1") + if err != nil { + t.Fatal(err) + } + if e.admission.active[interactiveClass] != 1 { + t.Fatalf("slot not held during read: active=%d", e.admission.active[interactiveClass]) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + if !closed { + t.Fatal("underlying rows were not closed") + } + if e.admission.active[interactiveClass] != 0 { + t.Fatalf("slot not released after close: active=%d", e.admission.active[interactiveClass]) + } +} + +// Closing twice releases the slot once, so a defensive double close cannot +// drive the admission count negative. +func TestAdmittedReaderReleasesItsSlotOnceOnDoubleClose(t *testing.T) { + e := newAdmissionFixture(t, 2, 0, 4) + closed := false + reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: &closed}}) + rows, err := reader.QueryContext(t.Context(), "SELECT 1") + if err != nil { + t.Fatal(err) + } + _ = rows.Close() + _ = rows.Close() + if e.admission.active[interactiveClass] != 0 { + t.Fatalf("double close changed the count: active=%d", e.admission.active[interactiveClass]) + } +} + +// When the interactive lane is full the routed read is shed as ErrBusy without +// ever reaching the engine, instead of oversubscribing the pool. +func TestAdmittedReaderShedsWhenTheLaneIsFull(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 1) + e.admission.bounds[interactiveClass] = 0 + if err := e.admission.acquire(t.Context(), interactiveClass); err != nil { + t.Fatal(err) + } + queried := false + reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: new(bool)}, queried: &queried}) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + rows, err := reader.QueryContext(ctx, "SELECT 1") + if !errors.Is(err, ErrBusy) { + t.Fatalf("err=%v, want ErrBusy", err) + } + if rows != nil { + t.Fatal("rows returned on a shed read") + } + if queried { + t.Fatal("engine was queried on a shed read") + } + if e.InteractiveRefusals() == 0 { + t.Fatal("a shed read did not count as an interactive refusal") + } +} + +// A routed read transaction takes one slot for the whole callback and runs on +// the wrapped surface's own transaction, so a reader needing one consistent +// snapshot keeps it and is admitted as a unit rather than per sub-read. +func TestAdmittedReadTransactionTakesOneSlotForTheWholeCallback(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 4) + ran := false + reader := e.AdmitReader(fakeReader{txRan: &ran}).(queryrows.ReadTransactioner) + err := reader.WithReadTransaction(t.Context(), func(queryrows.Queryer) error { + if e.admission.active[interactiveClass] != 1 { + t.Fatalf("slot not held during the callback: active=%d", e.admission.active[interactiveClass]) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if !ran { + t.Fatal("the wrapped read transaction did not run") + } + if e.admission.active[interactiveClass] != 0 { + t.Fatalf("slot not released after the callback: active=%d", e.admission.active[interactiveClass]) + } +} + +// The wrapper forwards the engine's default namespace, which annotation reads +// probe through an optional interface. +func TestAdmittedReaderForwardsTheDefaultNamespace(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 4) + reader := e.AdmitReader(fakeReader{namespace: "edge"}) + ns, ok := reader.(interface{ DefaultNamespace() string }) + if !ok { + t.Fatal("wrapper does not expose DefaultNamespace") + } + if got := ns.DefaultNamespace(); got != "edge" { + t.Fatalf("DefaultNamespace=%q, want edge", got) + } +} From c409a8185f7ce86bbb0b76b224d0069252e70d14 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 18 Nov 2026 02:39:23 -0800 Subject: [PATCH 2/2] perf(query): keep trace detail on the raw engine to avoid nested admission Trace detail is already admitted by the cached-trace run, so routing its candidate and log reads through the admitting observability reader acquired a second interactive slot while holding the first and could deadlock a one-slot pool. Use the raw engine for every reader reached through the executor (the dashboard rollup reader and trace detail) and admit only the readers that bypass it (the MCP observability tools and the annotation route). --- cmd/pitot/main.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cmd/pitot/main.go b/cmd/pitot/main.go index 5a2eb983..728a8d7d 100644 --- a/cmd/pitot/main.go +++ b/cmd/pitot/main.go @@ -299,10 +299,14 @@ func main() { // Route both HTTP and MCP reads through Duck's retrying adapter. Passing the // raw *sql.DB here bypassed the Telemetry maintenance-race protection. panels := query.NewExecutor(q, cfg.RetentionDays) - // The observability readers call the engine directly, so route them through - // the executor's interactive admission lane; otherwise they oversubscribe the - // read pool and starve admitted panels under a saturating dashboard. - queries := observability.New(panels.AdmitReader(q), q, cfg.RetentionDays) + // Observability reads reached through the executor (the dashboard panels' + // rollup reader and trace detail) are already admitted by the executor, so + // they use the raw engine; admitting them again would nest a second acquire + // and deadlock a one-slot pool. Reads that bypass the executor (the MCP + // observability tools, and the annotation route below) are admitted through + // the interactive lane so they cannot oversubscribe the read pool. + queries := observability.New(q, q, cfg.RetentionDays) + admittedQueries := observability.New(panels.AdmitReader(q), q, cfg.RetentionDays) writer.SetAdmission(repository.CompactionDebt, panels.InteractiveRefusals) go func() { err := writer.Run(ctx) @@ -316,10 +320,7 @@ func main() { }() q.SetReadPressure(panels.Pressure) go q.RunRollups(ctx) - // Panel rollup reads already run inside an admitted panel slot, so the rollup - // reader uses the raw engine. Admitting it again would make a one-slot pool - // deadlock: the panel holds the only slot while its nested read waits for one. - panels.SetRollupReader(observability.New(q, q, cfg.RetentionDays)) + panels.SetRollupReader(queries) panels.SetCacheTTL(cfg.QueryCacheTTL) api.RegisterQueryRoutes(e, panels) api.RegisterPitotQLRoutes(e, q, panels) @@ -397,7 +398,7 @@ func main() { // The model executes the same standard MCP tools exposed to external clients. // The internal connection is in-memory, so the single binary has no HTTP // self-call, shared secret, or sidecar runtime. - mcpServer := mcp.NewWithIntelligence(queries, dashboards, panels, detector, version) + mcpServer := mcp.NewWithIntelligence(admittedQueries, dashboards, panels, detector, version) if cfg.MCPEnabled { mcpResourceURL := cfg.MCPResourceURL() mcpAuthorization, err := api.NewMCPAuthorization(