diff --git a/cmd/pitot/main.go b/cmd/pitot/main.go index 63786a91..728a8d7d 100644 --- a/cmd/pitot/main.go +++ b/cmd/pitot/main.go @@ -298,8 +298,15 @@ func main() { // typed query kernel through deterministic HTTP or standard MCP tools. // Route both HTTP and MCP reads through Duck's retrying adapter. Passing the // raw *sql.DB here bypassed the Telemetry maintenance-race protection. - queries := observability.New(q, q, cfg.RetentionDays) panels := query.NewExecutor(q, cfg.RetentionDays) + // Observability reads reached through the executor (the dashboard panels' + // rollup reader and trace detail) are already admitted by the executor, so + // they use the raw engine; admitting them again would nest a second acquire + // and deadlock a one-slot pool. Reads that bypass the executor (the MCP + // observability tools, and the annotation route below) are admitted through + // the interactive lane so they cannot oversubscribe the read pool. + queries := observability.New(q, q, cfg.RetentionDays) + admittedQueries := observability.New(panels.AdmitReader(q), q, cfg.RetentionDays) writer.SetAdmission(repository.CompactionDebt, panels.InteractiveRefusals) go func() { err := writer.Run(ctx) @@ -317,7 +324,7 @@ func main() { panels.SetCacheTTL(cfg.QueryCacheTTL) api.RegisterQueryRoutes(e, panels) api.RegisterPitotQLRoutes(e, q, panels) - api.RegisterAnnotationRoutes(e, annotations.New(q)) + api.RegisterAnnotationRoutes(e, annotations.New(panels.AdmitReader(q))) api.RegisterTraceRoutes(e, panels.CachedTraces(queries), cfg.RetentionDays) api.RegisterIntelligenceRoutes(e, detector) dashboards := dashboard.New(sqlite.DB, panels) @@ -391,7 +398,7 @@ func main() { // The model executes the same standard MCP tools exposed to external clients. // The internal connection is in-memory, so the single binary has no HTTP // self-call, shared secret, or sidecar runtime. - mcpServer := mcp.NewWithIntelligence(queries, dashboards, panels, detector, version) + mcpServer := mcp.NewWithIntelligence(admittedQueries, dashboards, panels, detector, version) if cfg.MCPEnabled { mcpResourceURL := cfg.MCPResourceURL() mcpAuthorization, err := api.NewMCPAuthorization( diff --git a/internal/api/annotations.go b/internal/api/annotations.go index 9b5c2657..0087b586 100644 --- a/internal/api/annotations.go +++ b/internal/api/annotations.go @@ -7,6 +7,7 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/pitot/internal/annotations" + "github.com/labstack/pitot/internal/query" ) type AnnotationReader interface { @@ -21,16 +22,18 @@ func RegisterAnnotationRoutes(e *echo.Echo, service AnnotationReader) { } out, err := service.Read(c.Request().Context(), req) if err != nil { - return annotationError(err) + return annotationError(c, err) } return c.JSON(http.StatusOK, out) }, RequireCapability(ReadTelemetry)) } -func annotationError(err error) error { +func annotationError(c *echo.Context, err error) error { switch { case errors.Is(err, annotations.ErrRequest): return echo.NewHTTPError(http.StatusBadRequest, err.Error()).Wrap(err) + case errors.Is(err, query.ErrBusy): + return busyError(c) case errors.Is(err, context.DeadlineExceeded): return echo.NewHTTPError(http.StatusGatewayTimeout, "Annotations took longer than 10 seconds").Wrap(err) case errors.Is(err, context.Canceled): diff --git a/internal/api/annotations_test.go b/internal/api/annotations_test.go index fb645c12..9054c815 100644 --- a/internal/api/annotations_test.go +++ b/internal/api/annotations_test.go @@ -12,6 +12,7 @@ import ( "github.com/labstack/echo/v5" "github.com/labstack/pitot/internal/annotations" + "github.com/labstack/pitot/internal/query" ) type annotationFake struct{} @@ -21,16 +22,20 @@ func (annotationFake) Read(context.Context, annotations.Request) (annotations.Re } func TestAnnotationErrorsWrapped(t *testing.T) { + e := echo.New() + newCtx := func() *echo.Context { + return e.NewContext(httptest.NewRequest(http.MethodPost, "/", nil), httptest.NewRecorder()) + } original := errors.New("storage failed") for _, cause := range []error{annotations.ErrRequest, context.DeadlineExceeded, original} { wrapped := fmt.Errorf("annotation query: %w", cause) - got := annotationError(wrapped) + got := annotationError(newCtx(), wrapped) var httpErr *echo.HTTPError if !errors.As(got, &httpErr) || !errors.Is(got, cause) { t.Fatalf("cause lost: %v", got) } } - if got := annotationError(fmt.Errorf("read: %w", context.Canceled)); got != nil { + if got := annotationError(newCtx(), fmt.Errorf("read: %w", context.Canceled)); got != nil { t.Fatalf("cancellation: %v", got) } } @@ -50,6 +55,7 @@ func TestAnnotationErrorMappings(t *testing.T) { {"request", annotations.ErrRequest, 400, annotations.ErrRequest.Error()}, {"deadline", context.DeadlineExceeded, 504, "Annotations took longer than 10 seconds"}, {"canceled", context.Canceled, 200, ""}, + {"busy", query.ErrBusy, 503, query.BusyMessage}, {"unknown", errors.New("storage failed"), 500, "annotations unavailable"}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/internal/config/sizing.go b/internal/config/sizing.go index d42a541f..f7b03dc6 100644 --- a/internal/config/sizing.go +++ b/internal/config/sizing.go @@ -51,8 +51,10 @@ const ( // QueryAdmission derives the executor's read admission from the pool: // slots reads hold connections at once. reserved slots serve interactive // reads only; queue bounds dashboard batch tickets (interactive waits are 2*slots). -// A one-slot pool has no reservation so panels can still execute. A pool of four or more keeps two connections -// for annotations and readiness checks, which do not queue behind them. +// A one-slot pool has no reservation so panels can still execute. A pool of four +// or more keeps two connections outside the slots for readiness checks and other +// direct engine reads (such as the panel-internal rollup reader) that must not +// queue behind admitted panels. func QueryAdmission(pool int) (slots, reserved, queue int) { slots = max(1, pool) if pool >= 4 { diff --git a/internal/query/admit_reader.go b/internal/query/admit_reader.go new file mode 100644 index 00000000..160b8d3d --- /dev/null +++ b/internal/query/admit_reader.go @@ -0,0 +1,114 @@ +package query + +import ( + "context" + "errors" + "sync" + + "github.com/labstack/pitot/internal/queryrows" +) + +// AdmitReader wraps a read surface so every read it issues passes through the +// interactive admission lane. The observability readers, annotations, and +// variable-option queries otherwise call the engine directly and compete for +// the read pool without being admitted, so a saturating dashboard oversubscribes +// the pool, starves admitted panels, and defeats the deadline shedding the pool +// size is meant to enforce. Routing those reads through admission restores the +// invariant that no more reads run concurrently than the pool can serve, and +// turns excess load into a bounded ErrBusy shed rather than unbounded queueing +// inside the engine. +// +// The admission slot is held for the lifetime of the returned rows and released +// when they close, mirroring how the engine holds a pool connection until its +// rows close. Callers already close every result set before issuing the next +// read, so one interactive slot is never held across acquiring another. +func (e *Executor) AdmitReader(inner queryrows.Queryer) queryrows.Queryer { + return &admittingQueryer{inner: inner, exec: e} +} + +type admittingQueryer struct { + inner queryrows.Queryer + exec *Executor +} + +func (a *admittingQueryer) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { + release, err := a.exec.admitInteractive(ctx) + if err != nil { + return nil, err + } + rows, err := a.inner.QueryContext(ctx, query, args...) + if err != nil { + release() + return nil, err + } + return &admittedRows{Rows: rows, release: release}, nil +} + +// WithReadTransaction admits the whole callback as one unit and runs it on the +// wrapped surface's own read transaction, so a reader that needs a single +// consistent snapshot (annotations reads deploys, anomalies, and the watermark +// together) keeps it and takes one slot for the run rather than one per +// sub-read. A surface without transaction support runs the callback against +// this admitting queryer instead, so its reads are still admitted. +func (a *admittingQueryer) WithReadTransaction(ctx context.Context, read func(queryrows.Queryer) error) error { + rt, ok := a.inner.(queryrows.ReadTransactioner) + if !ok { + return read(a) + } + release, err := a.exec.admitInteractive(ctx) + if err != nil { + return err + } + defer release() + return rt.WithReadTransaction(ctx, read) +} + +// DefaultNamespace forwards to the wrapped surface when it exposes one, matching +// the optional-interface probe callers use, and is empty otherwise. +func (a *admittingQueryer) DefaultNamespace() string { + if d, ok := a.inner.(interface{ DefaultNamespace() string }); ok { + return d.DefaultNamespace() + } + return "" +} + +// admitInteractive takes one interactive slot, bounding the wait by the batch +// timeout as the exemplar and trace-detail path does, so a deadline-less caller +// (a background or MCP read) cannot sit in the bounded queue indefinitely. A +// full queue or an expired wait is counted as an interactive refusal and +// returned as ErrBusy, except when the caller's own context ended first. These +// refusals share the executor's interactive-pressure counter, so a flood of +// routed observability or annotation reads also feeds ingest backpressure — +// intended, since interactive read starvation is exactly when ingest should +// yield. +func (e *Executor) admitInteractive(ctx context.Context) (func(), error) { + wait, cancel := context.WithTimeout(ctx, e.batchTimeout) + defer cancel() + err := e.acquireClass(wait, interactiveClass) + if errors.Is(err, errQueueFull) || errors.Is(err, context.DeadlineExceeded) { + e.interactiveRefusals.Add(1) + } + switch { + case err == nil: + return func() { e.admission.release(interactiveClass) }, nil + case ctx.Err() != nil: + return nil, ctx.Err() + default: + return nil, ErrBusy + } +} + +// admittedRows releases the admission slot when the wrapped rows close. Close is +// idempotent, so a double close or a close after exhaustion releases the slot +// exactly once. +type admittedRows struct { + queryrows.Rows + release func() + once sync.Once +} + +func (r *admittedRows) Close() error { + err := r.Rows.Close() + r.once.Do(r.release) + return err +} diff --git a/internal/query/admit_reader_test.go b/internal/query/admit_reader_test.go new file mode 100644 index 00000000..116ba790 --- /dev/null +++ b/internal/query/admit_reader_test.go @@ -0,0 +1,155 @@ +package query + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/labstack/pitot/internal/queryrows" +) + +type fakeRows struct{ closed *bool } + +func (fakeRows) Columns() ([]string, error) { return nil, nil } +func (fakeRows) Err() error { return nil } +func (fakeRows) Next() bool { return false } +func (fakeRows) Scan(...any) error { return nil } +func (r fakeRows) Close() error { *r.closed = true; return nil } + +type fakeReader struct { + namespace string + rows queryrows.Rows + queried *bool + txRan *bool +} + +func (f fakeReader) QueryContext(context.Context, string, ...any) (queryrows.Rows, error) { + if f.queried != nil { + *f.queried = true + } + return f.rows, nil +} + +func (f fakeReader) DefaultNamespace() string { return f.namespace } + +func (f fakeReader) WithReadTransaction(ctx context.Context, read func(queryrows.Queryer) error) error { + if f.txRan != nil { + *f.txRan = true + } + return read(f) +} + +func newAdmissionFixture(t *testing.T, slots, reserved, queue int) *Executor { + t.Helper() + e := NewExecutor(pooledEngine{pool: 8}, 30) + t.Cleanup(e.Close) + e.admission = newReadAdmission(slots, reserved, queue) + return e +} + +// A routed read holds its admission slot until the rows close, then frees it, so +// a bypass reader can no longer occupy the pool after it has finished reading. +func TestAdmittedReaderHoldsItsSlotUntilRowsClose(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 4) + closed := false + reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: &closed}}) + rows, err := reader.QueryContext(t.Context(), "SELECT 1") + if err != nil { + t.Fatal(err) + } + if e.admission.active[interactiveClass] != 1 { + t.Fatalf("slot not held during read: active=%d", e.admission.active[interactiveClass]) + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + if !closed { + t.Fatal("underlying rows were not closed") + } + if e.admission.active[interactiveClass] != 0 { + t.Fatalf("slot not released after close: active=%d", e.admission.active[interactiveClass]) + } +} + +// Closing twice releases the slot once, so a defensive double close cannot +// drive the admission count negative. +func TestAdmittedReaderReleasesItsSlotOnceOnDoubleClose(t *testing.T) { + e := newAdmissionFixture(t, 2, 0, 4) + closed := false + reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: &closed}}) + rows, err := reader.QueryContext(t.Context(), "SELECT 1") + if err != nil { + t.Fatal(err) + } + _ = rows.Close() + _ = rows.Close() + if e.admission.active[interactiveClass] != 0 { + t.Fatalf("double close changed the count: active=%d", e.admission.active[interactiveClass]) + } +} + +// When the interactive lane is full the routed read is shed as ErrBusy without +// ever reaching the engine, instead of oversubscribing the pool. +func TestAdmittedReaderShedsWhenTheLaneIsFull(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 1) + e.admission.bounds[interactiveClass] = 0 + if err := e.admission.acquire(t.Context(), interactiveClass); err != nil { + t.Fatal(err) + } + queried := false + reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: new(bool)}, queried: &queried}) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + rows, err := reader.QueryContext(ctx, "SELECT 1") + if !errors.Is(err, ErrBusy) { + t.Fatalf("err=%v, want ErrBusy", err) + } + if rows != nil { + t.Fatal("rows returned on a shed read") + } + if queried { + t.Fatal("engine was queried on a shed read") + } + if e.InteractiveRefusals() == 0 { + t.Fatal("a shed read did not count as an interactive refusal") + } +} + +// A routed read transaction takes one slot for the whole callback and runs on +// the wrapped surface's own transaction, so a reader needing one consistent +// snapshot keeps it and is admitted as a unit rather than per sub-read. +func TestAdmittedReadTransactionTakesOneSlotForTheWholeCallback(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 4) + ran := false + reader := e.AdmitReader(fakeReader{txRan: &ran}).(queryrows.ReadTransactioner) + err := reader.WithReadTransaction(t.Context(), func(queryrows.Queryer) error { + if e.admission.active[interactiveClass] != 1 { + t.Fatalf("slot not held during the callback: active=%d", e.admission.active[interactiveClass]) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if !ran { + t.Fatal("the wrapped read transaction did not run") + } + if e.admission.active[interactiveClass] != 0 { + t.Fatalf("slot not released after the callback: active=%d", e.admission.active[interactiveClass]) + } +} + +// The wrapper forwards the engine's default namespace, which annotation reads +// probe through an optional interface. +func TestAdmittedReaderForwardsTheDefaultNamespace(t *testing.T) { + e := newAdmissionFixture(t, 1, 0, 4) + reader := e.AdmitReader(fakeReader{namespace: "edge"}) + ns, ok := reader.(interface{ DefaultNamespace() string }) + if !ok { + t.Fatal("wrapper does not expose DefaultNamespace") + } + if got := ns.DefaultNamespace(); got != "edge" { + t.Fatalf("DefaultNamespace=%q, want edge", got) + } +}