Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions cmd/pitot/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,8 +298,15 @@ func main() {
// typed query kernel through deterministic HTTP or standard MCP tools.
// Route both HTTP and MCP reads through Duck's retrying adapter. Passing the
// raw *sql.DB here bypassed the Telemetry maintenance-race protection.
queries := observability.New(q, q, cfg.RetentionDays)
panels := query.NewExecutor(q, cfg.RetentionDays)
// Observability reads reached through the executor (the dashboard panels'
// rollup reader and trace detail) are already admitted by the executor, so
// they use the raw engine; admitting them again would nest a second acquire
// and deadlock a one-slot pool. Reads that bypass the executor (the MCP
// observability tools, and the annotation route below) are admitted through
// the interactive lane so they cannot oversubscribe the read pool.
queries := observability.New(q, q, cfg.RetentionDays)
admittedQueries := observability.New(panels.AdmitReader(q), q, cfg.RetentionDays)
writer.SetAdmission(repository.CompactionDebt, panels.InteractiveRefusals)
go func() {
err := writer.Run(ctx)
Expand All @@ -317,7 +324,7 @@ func main() {
panels.SetCacheTTL(cfg.QueryCacheTTL)
api.RegisterQueryRoutes(e, panels)
api.RegisterPitotQLRoutes(e, q, panels)
api.RegisterAnnotationRoutes(e, annotations.New(q))
api.RegisterAnnotationRoutes(e, annotations.New(panels.AdmitReader(q)))
api.RegisterTraceRoutes(e, panels.CachedTraces(queries), cfg.RetentionDays)
api.RegisterIntelligenceRoutes(e, detector)
dashboards := dashboard.New(sqlite.DB, panels)
Expand Down Expand Up @@ -391,7 +398,7 @@ func main() {
// The model executes the same standard MCP tools exposed to external clients.
// The internal connection is in-memory, so the single binary has no HTTP
// self-call, shared secret, or sidecar runtime.
mcpServer := mcp.NewWithIntelligence(queries, dashboards, panels, detector, version)
mcpServer := mcp.NewWithIntelligence(admittedQueries, dashboards, panels, detector, version)
if cfg.MCPEnabled {
mcpResourceURL := cfg.MCPResourceURL()
mcpAuthorization, err := api.NewMCPAuthorization(
Expand Down
7 changes: 5 additions & 2 deletions internal/api/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/labstack/echo/v5"
"github.com/labstack/pitot/internal/annotations"
"github.com/labstack/pitot/internal/query"
)

type AnnotationReader interface {
Expand All @@ -21,16 +22,18 @@ func RegisterAnnotationRoutes(e *echo.Echo, service AnnotationReader) {
}
out, err := service.Read(c.Request().Context(), req)
if err != nil {
return annotationError(err)
return annotationError(c, err)
}
return c.JSON(http.StatusOK, out)
}, RequireCapability(ReadTelemetry))
}

func annotationError(err error) error {
func annotationError(c *echo.Context, err error) error {
switch {
case errors.Is(err, annotations.ErrRequest):
return echo.NewHTTPError(http.StatusBadRequest, err.Error()).Wrap(err)
case errors.Is(err, query.ErrBusy):
return busyError(c)
case errors.Is(err, context.DeadlineExceeded):
return echo.NewHTTPError(http.StatusGatewayTimeout, "Annotations took longer than 10 seconds").Wrap(err)
case errors.Is(err, context.Canceled):
Expand Down
10 changes: 8 additions & 2 deletions internal/api/annotations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/labstack/echo/v5"
"github.com/labstack/pitot/internal/annotations"
"github.com/labstack/pitot/internal/query"
)

type annotationFake struct{}
Expand All @@ -21,16 +22,20 @@ func (annotationFake) Read(context.Context, annotations.Request) (annotations.Re
}

func TestAnnotationErrorsWrapped(t *testing.T) {
e := echo.New()
newCtx := func() *echo.Context {
return e.NewContext(httptest.NewRequest(http.MethodPost, "/", nil), httptest.NewRecorder())
}
original := errors.New("storage failed")
for _, cause := range []error{annotations.ErrRequest, context.DeadlineExceeded, original} {
wrapped := fmt.Errorf("annotation query: %w", cause)
got := annotationError(wrapped)
got := annotationError(newCtx(), wrapped)
var httpErr *echo.HTTPError
if !errors.As(got, &httpErr) || !errors.Is(got, cause) {
t.Fatalf("cause lost: %v", got)
}
}
if got := annotationError(fmt.Errorf("read: %w", context.Canceled)); got != nil {
if got := annotationError(newCtx(), fmt.Errorf("read: %w", context.Canceled)); got != nil {
t.Fatalf("cancellation: %v", got)
}
}
Expand All @@ -50,6 +55,7 @@ func TestAnnotationErrorMappings(t *testing.T) {
{"request", annotations.ErrRequest, 400, annotations.ErrRequest.Error()},
{"deadline", context.DeadlineExceeded, 504, "Annotations took longer than 10 seconds"},
{"canceled", context.Canceled, 200, ""},
{"busy", query.ErrBusy, 503, query.BusyMessage},
{"unknown", errors.New("storage failed"), 500, "annotations unavailable"},
} {
t.Run(tc.name, func(t *testing.T) {
Expand Down
6 changes: 4 additions & 2 deletions internal/config/sizing.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,10 @@ const (
// QueryAdmission derives the executor's read admission from the pool:
// slots reads hold connections at once. reserved slots serve interactive
// reads only; queue bounds dashboard batch tickets (interactive waits are 2*slots).
// A one-slot pool has no reservation so panels can still execute. A pool of four or more keeps two connections
// for annotations and readiness checks, which do not queue behind them.
// A one-slot pool has no reservation so panels can still execute. A pool of four
// or more keeps two connections outside the slots for readiness checks and other
// direct engine reads (such as the panel-internal rollup reader) that must not
// queue behind admitted panels.
func QueryAdmission(pool int) (slots, reserved, queue int) {
slots = max(1, pool)
if pool >= 4 {
Expand Down
114 changes: 114 additions & 0 deletions internal/query/admit_reader.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package query

import (
"context"
"errors"
"sync"

"github.com/labstack/pitot/internal/queryrows"
)

// AdmitReader wraps a read surface so every read it issues passes through the
// interactive admission lane. The observability readers, annotations, and
// variable-option queries otherwise call the engine directly and compete for
// the read pool without being admitted, so a saturating dashboard oversubscribes
// the pool, starves admitted panels, and defeats the deadline shedding the pool
// size is meant to enforce. Routing those reads through admission restores the
// invariant that no more reads run concurrently than the pool can serve, and
// turns excess load into a bounded ErrBusy shed rather than unbounded queueing
// inside the engine.
//
// The admission slot is held for the lifetime of the returned rows and released
// when they close, mirroring how the engine holds a pool connection until its
// rows close. Callers already close every result set before issuing the next
// read, so one interactive slot is never held across acquiring another.
func (e *Executor) AdmitReader(inner queryrows.Queryer) queryrows.Queryer {
return &admittingQueryer{inner: inner, exec: e}
}

type admittingQueryer struct {
inner queryrows.Queryer
exec *Executor
}

func (a *admittingQueryer) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) {
release, err := a.exec.admitInteractive(ctx)
if err != nil {
return nil, err
}
rows, err := a.inner.QueryContext(ctx, query, args...)
if err != nil {
release()
return nil, err
}
return &admittedRows{Rows: rows, release: release}, nil
}

// WithReadTransaction admits the whole callback as one unit and runs it on the
// wrapped surface's own read transaction, so a reader that needs a single
// consistent snapshot (annotations reads deploys, anomalies, and the watermark
// together) keeps it and takes one slot for the run rather than one per
// sub-read. A surface without transaction support runs the callback against
// this admitting queryer instead, so its reads are still admitted.
func (a *admittingQueryer) WithReadTransaction(ctx context.Context, read func(queryrows.Queryer) error) error {
rt, ok := a.inner.(queryrows.ReadTransactioner)
if !ok {
return read(a)
}
release, err := a.exec.admitInteractive(ctx)
if err != nil {
return err
}
defer release()
return rt.WithReadTransaction(ctx, read)
}

// DefaultNamespace forwards to the wrapped surface when it exposes one, matching
// the optional-interface probe callers use, and is empty otherwise.
func (a *admittingQueryer) DefaultNamespace() string {
if d, ok := a.inner.(interface{ DefaultNamespace() string }); ok {
return d.DefaultNamespace()
}
return ""
}

// admitInteractive takes one interactive slot, bounding the wait by the batch
// timeout as the exemplar and trace-detail path does, so a deadline-less caller
// (a background or MCP read) cannot sit in the bounded queue indefinitely. A
// full queue or an expired wait is counted as an interactive refusal and
// returned as ErrBusy, except when the caller's own context ended first. These
// refusals share the executor's interactive-pressure counter, so a flood of
// routed observability or annotation reads also feeds ingest backpressure —
// intended, since interactive read starvation is exactly when ingest should
// yield.
func (e *Executor) admitInteractive(ctx context.Context) (func(), error) {
wait, cancel := context.WithTimeout(ctx, e.batchTimeout)
defer cancel()
err := e.acquireClass(wait, interactiveClass)
if errors.Is(err, errQueueFull) || errors.Is(err, context.DeadlineExceeded) {
e.interactiveRefusals.Add(1)
}
switch {
case err == nil:
return func() { e.admission.release(interactiveClass) }, nil
case ctx.Err() != nil:
return nil, ctx.Err()
default:
return nil, ErrBusy
}
}

// admittedRows releases the admission slot when the wrapped rows close. Close is
// idempotent, so a double close or a close after exhaustion releases the slot
// exactly once.
type admittedRows struct {
queryrows.Rows
release func()
once sync.Once
}

func (r *admittedRows) Close() error {
err := r.Rows.Close()
r.once.Do(r.release)
return err
}
155 changes: 155 additions & 0 deletions internal/query/admit_reader_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
package query

import (
"context"
"errors"
"testing"
"time"

"github.com/labstack/pitot/internal/queryrows"
)

type fakeRows struct{ closed *bool }

func (fakeRows) Columns() ([]string, error) { return nil, nil }
func (fakeRows) Err() error { return nil }
func (fakeRows) Next() bool { return false }
func (fakeRows) Scan(...any) error { return nil }
func (r fakeRows) Close() error { *r.closed = true; return nil }

type fakeReader struct {
namespace string
rows queryrows.Rows
queried *bool
txRan *bool
}

func (f fakeReader) QueryContext(context.Context, string, ...any) (queryrows.Rows, error) {
if f.queried != nil {
*f.queried = true
}
return f.rows, nil
}

func (f fakeReader) DefaultNamespace() string { return f.namespace }

func (f fakeReader) WithReadTransaction(ctx context.Context, read func(queryrows.Queryer) error) error {
if f.txRan != nil {
*f.txRan = true
}
return read(f)
}

func newAdmissionFixture(t *testing.T, slots, reserved, queue int) *Executor {
t.Helper()
e := NewExecutor(pooledEngine{pool: 8}, 30)
t.Cleanup(e.Close)
e.admission = newReadAdmission(slots, reserved, queue)
return e
}

// A routed read holds its admission slot until the rows close, then frees it, so
// a bypass reader can no longer occupy the pool after it has finished reading.
func TestAdmittedReaderHoldsItsSlotUntilRowsClose(t *testing.T) {
e := newAdmissionFixture(t, 1, 0, 4)
closed := false
reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: &closed}})
rows, err := reader.QueryContext(t.Context(), "SELECT 1")
if err != nil {
t.Fatal(err)
}
if e.admission.active[interactiveClass] != 1 {
t.Fatalf("slot not held during read: active=%d", e.admission.active[interactiveClass])
}
if err := rows.Close(); err != nil {
t.Fatal(err)
}
if !closed {
t.Fatal("underlying rows were not closed")
}
if e.admission.active[interactiveClass] != 0 {
t.Fatalf("slot not released after close: active=%d", e.admission.active[interactiveClass])
}
}

// Closing twice releases the slot once, so a defensive double close cannot
// drive the admission count negative.
func TestAdmittedReaderReleasesItsSlotOnceOnDoubleClose(t *testing.T) {
e := newAdmissionFixture(t, 2, 0, 4)
closed := false
reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: &closed}})
rows, err := reader.QueryContext(t.Context(), "SELECT 1")
if err != nil {
t.Fatal(err)
}
_ = rows.Close()
_ = rows.Close()
if e.admission.active[interactiveClass] != 0 {
t.Fatalf("double close changed the count: active=%d", e.admission.active[interactiveClass])
}
}

// When the interactive lane is full the routed read is shed as ErrBusy without
// ever reaching the engine, instead of oversubscribing the pool.
func TestAdmittedReaderShedsWhenTheLaneIsFull(t *testing.T) {
e := newAdmissionFixture(t, 1, 0, 1)
e.admission.bounds[interactiveClass] = 0
if err := e.admission.acquire(t.Context(), interactiveClass); err != nil {
t.Fatal(err)
}
queried := false
reader := e.AdmitReader(fakeReader{rows: fakeRows{closed: new(bool)}, queried: &queried})
ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond)
defer cancel()
rows, err := reader.QueryContext(ctx, "SELECT 1")
if !errors.Is(err, ErrBusy) {
t.Fatalf("err=%v, want ErrBusy", err)
}
if rows != nil {
t.Fatal("rows returned on a shed read")
}
if queried {
t.Fatal("engine was queried on a shed read")
}
if e.InteractiveRefusals() == 0 {
t.Fatal("a shed read did not count as an interactive refusal")
}
}

// A routed read transaction takes one slot for the whole callback and runs on
// the wrapped surface's own transaction, so a reader needing one consistent
// snapshot keeps it and is admitted as a unit rather than per sub-read.
func TestAdmittedReadTransactionTakesOneSlotForTheWholeCallback(t *testing.T) {
e := newAdmissionFixture(t, 1, 0, 4)
ran := false
reader := e.AdmitReader(fakeReader{txRan: &ran}).(queryrows.ReadTransactioner)
err := reader.WithReadTransaction(t.Context(), func(queryrows.Queryer) error {
if e.admission.active[interactiveClass] != 1 {
t.Fatalf("slot not held during the callback: active=%d", e.admission.active[interactiveClass])
}
return nil
})
if err != nil {
t.Fatal(err)
}
if !ran {
t.Fatal("the wrapped read transaction did not run")
}
if e.admission.active[interactiveClass] != 0 {
t.Fatalf("slot not released after the callback: active=%d", e.admission.active[interactiveClass])
}
}

// The wrapper forwards the engine's default namespace, which annotation reads
// probe through an optional interface.
func TestAdmittedReaderForwardsTheDefaultNamespace(t *testing.T) {
e := newAdmissionFixture(t, 1, 0, 4)
reader := e.AdmitReader(fakeReader{namespace: "edge"})
ns, ok := reader.(interface{ DefaultNamespace() string })
if !ok {
t.Fatal("wrapper does not expose DefaultNamespace")
}
if got := ns.DefaultNamespace(); got != "edge" {
t.Fatalf("DefaultNamespace=%q, want edge", got)
}
}
Loading