From 461cebf0f4c06281d392cbe4dfdc4aec7e40b04e Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 11:26:51 +0100 Subject: [PATCH 1/8] fix(webapp): make the Queues hero charts environment-wide MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four charts above the queues table reused the loader's already-paginated queue array as a ClickHouse `queue IN (...)` filter, so they aggregated over at most the 25 queues on the current page. Paging or re-sorting changed the values, and a name search that matched nothing blanked the whole chart row. They now read `env_metrics`, the environment-level rollup that already exists for this (the built-in Queues dashboard and the health report read it), which is both correct and queue-count-independent: no `GROUP BY queue` across an entire environment and no client-side summing. Two related fixes ride along: Scheduling delay and throttling are event-driven, so at the 10-second bucket a short range picks, most buckets hold no samples at all and were drawn as 0ms — measured at 232 of 349 buckets over an hour. TRQL grows a `minBucketSeconds` floor, plumbed through the metric resource route, and the hero tiles set 60s (one floor for all four, since the shared hover crosshair needs identical x-axes). Buckets that still have no samples now render as a gap rather than a dive to zero. Note that `wait_ms_count` only counts `wait_ms > 0`, so "nothing started" and "everything started instantly" are indistinguishable in storage; both read as a gap. Recharts was resolving victory-vendor's CJS entry on the server and its ESM entry in the browser. Those bundle different d3-shape builds — the CJS one predates d3-path's digit rounding — so every server-rendered curve carried full-precision coordinates while the client rounded to 3 decimals, and React reported a hydration mismatch on every chart. Bundling recharts for SSR makes both sides resolve the same ESM build. --- .../queues-page-environment-wide-charts.md | 6 + .../app/hooks/useMetricResourceQuery.ts | 19 ++- .../route.tsx | 123 ++++++------------ apps/webapp/app/routes/resources.metric.tsx | 3 + .../app/services/queryService.server.ts | 26 ++-- apps/webapp/vite.config.ts | 2 + .../clickhouse/src/client/tsql.ts | 6 + internal-packages/tsql/src/index.ts | 9 ++ internal-packages/tsql/src/query/printer.ts | 6 +- .../tsql/src/query/printer_context.ts | 19 ++- .../tsql/src/query/time_buckets.test.ts | 48 ++++++- .../tsql/src/query/time_buckets.ts | 46 ++++++- 12 files changed, 209 insertions(+), 104 deletions(-) create mode 100644 .server-changes/queues-page-environment-wide-charts.md diff --git a/.server-changes/queues-page-environment-wide-charts.md b/.server-changes/queues-page-environment-wide-charts.md new file mode 100644 index 00000000000..bb3d8ea5da8 --- /dev/null +++ b/.server-changes/queues-page-environment-wide-charts.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +The four charts at the top of the Queues page now always cover the whole environment, so paging through or re-sorting your queues no longer changes them. The scheduling delay chart also leaves a gap where no runs started, instead of dropping to zero. diff --git a/apps/webapp/app/hooks/useMetricResourceQuery.ts b/apps/webapp/app/hooks/useMetricResourceQuery.ts index a4f26bfd9d2..2ada75735bd 100644 --- a/apps/webapp/app/hooks/useMetricResourceQuery.ts +++ b/apps/webapp/app/hooks/useMetricResourceQuery.ts @@ -21,6 +21,8 @@ export type MetricResourceQueryOptions = { defaultPeriod: string; queues?: string[]; fillGaps?: boolean; + /** Floor for the query's bucket width, for series too sparse to read at the range's width. */ + minBucketSeconds?: number; refreshIntervalMs?: number; }; @@ -56,6 +58,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO environmentId, defaultPeriod, fillGaps, + minBucketSeconds, refreshIntervalMs = 60_000, } = opts; const { period, from, to } = opts.timeRange; @@ -71,10 +74,22 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO from ?? "", to ?? "", fillGaps ? 1 : 0, + minBucketSeconds ?? "", queuesKey ?? "", query, ].join("|"), - [organizationId, projectId, environmentId, resolvedPeriod, from, to, fillGaps, queuesKey, query] + [ + organizationId, + projectId, + environmentId, + resolvedPeriod, + from, + to, + fillGaps, + minBucketSeconds, + queuesKey, + query, + ] ); const [rows, setRows] = useState( @@ -112,6 +127,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO organizationId, projectId, environmentId, + ...(minBucketSeconds !== undefined ? { minBucketSeconds } : {}), ...(queuesKey !== undefined ? { queues: queuesKey.split(",") } : {}), }), signal: controller.signal, @@ -142,6 +158,7 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO from, to, fillGaps, + minBucketSeconds, organizationId, projectId, environmentId, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 47ab9581695..a1855ede8f6 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -391,13 +391,6 @@ function QueuesWithMetricsView() { const metricsByQueue = metrics?.byQueue ?? {}; - // The four header charts mirror exactly the queue set the table is showing (post-search, - // post-pagination). These are the `task/`-prefixed queue_name values the queue_metrics table - // stores, so the client-side tile queries scope to the same rows the loader listed. - const chartQueueNames = success - ? queues.map((q) => (q.type === "task" ? `task/${q.name}` : q.name)) - : []; - const organization = useOrganization(); const project = useProject(); const env = useEnvironment(); @@ -635,13 +628,7 @@ function QueuesWithMetricsView() { ) : null} - {/* Env saturation, Backlog, Scheduling delay p95, Throttled viz — full-size, synced, - drag-to-zoom line charts (Agent page pattern). Four chart tiles: 2x2 below lg, 4-up - from lg, derived from the tile count. `kind="charts"` bakes the fixed row height. - Only when there are queues to chart: not-success states (engine-version, no tasks) and a - filtered-to-empty list leave chartQueueNames empty, where the tiles would just render - four "No activity" cards above the blank state. */} - {chartQueueNames.length > 0 ? ( + {success && (hasFilters || totalQueues !== 0) ? ( {QUEUE_HEADER_TILES.map((tile) => ( @@ -649,7 +636,6 @@ function QueuesWithMetricsView() { key={tile.id} tile={tile} timeRange={timeRange} - queueNames={chartQueueNames} referenceLines={ tile.id === "saturation" ? [ @@ -1202,8 +1188,7 @@ export function QueueFilters() { type MetricTileRow = Record; -/** One charted point per time bucket, already aggregated across the visible queue set. */ -type TilePoint = { bucket: number; value: number }; +type TilePoint = { bucket: number; value: number | null }; // Inline colour swatch matching the chart's warning ("yellow") line — used in tooltip copy that // refers to that colour instead of naming it, so the swatch always matches the chart. @@ -1235,10 +1220,8 @@ type QueueHeaderTile = { /** Hover tooltip explaining the headline readout next to the title (e.g. what "9% of current * period" means). Without it the readout has no tooltip. */ totalTooltip?: string; - // Rows can be one-per-bucket (p95, throttled: aggregated across the set in ClickHouse) or - // one-per-(bucket, queue) (saturation, backlog: summed across the set here, since summing a - // gauge across queues can't be a flat aggregate without double-counting sub-buckets). Either - // way derive returns the per-bucket points the chart draws. + /** Turns one row per bucket into the per-bucket points the chart draws. A null value is a + * bucket the metric has nothing to say about, and the line breaks there rather than reading 0. */ derive: (rows: MetricTileRow[]) => { points: TilePoint[]; total: number; @@ -1257,39 +1240,19 @@ function tileTimeToMs(value: number | string | null): number { return Date.parse(s.endsWith("Z") ? s : `${s}Z`); } -// Sums a per-(bucket, queue) row set into one value per bucket. `read` pulls the queue's -// contribution; `envColumn`, when set, carries an env-wide column (identical across the set's -// rows in a bucket) through as the max, so saturation can divide by the env limit. -function sumByBucket( - rows: MetricTileRow[], - read: (row: MetricTileRow) => number, - envColumn?: string -): Array<{ bucket: number; sum: number; env: number }> { - const byBucket = new Map(); - for (const row of rows) { - const bucket = tileTimeToMs(row.t); - if (!Number.isFinite(bucket)) continue; - const entry = byBucket.get(bucket) ?? { sum: 0, env: 0 }; - entry.sum += read(row); - if (envColumn) entry.env = Math.max(entry.env, tileNumber(row[envColumn])); - byBucket.set(bucket, entry); - } - return [...byBucket.entries()] - .map(([bucket, { sum, env }]) => ({ bucket, sum, env })) - .sort((a, b) => a.bucket - b.bucket); +/** Peak of a series, ignoring the buckets it has nothing to say about. */ +function peakOf(points: TilePoint[]): number { + return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } -// Header tiles fetch their own TRQL query client-side (resources.metric) with fillGaps, scoped to -// the visible queue set (queue_metrics WHERE queue IN ). Saturation and backlog GROUP BY the -// queue too and sum here; p95 merges quantile states and throttled sums counters in ClickHouse. const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { id: "saturation", label: "Env saturation", description: ( <> - How much of the environment's concurrency these queues are using. Turns {" "} - above 100%, when they're into burst capacity. + How much of the environment's concurrency is in use. Turns above 100%, + when it's into burst capacity. ), color: "var(--color-queues)", @@ -1297,35 +1260,32 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-queues)", label: "Saturation" }, { color: "var(--color-warning)", label: "Over limit" }, ], - // Numerator: running summed across the visible set. Denominator: the env-wide limit (same for - // every queue in a bucket), so the line reads as the set's share of the environment capacity. - query: `SELECT timeBucket() AS t,\n queue,\n max(max_running) AS running,\n max(max_env_limit) AS env_limit\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t,\n max(max_env_running) AS running,\n max(max_env_limit) AS env_limit\nFROM env_metrics\nGROUP BY t\nORDER BY t`, formatValue: (v) => (v > 100 ? `${v}% — over the environment limit` : `${v}%`), formatAxis: (v) => `${v}%`, derive: (rows) => { - const points = sumByBucket(rows, (r) => tileNumber(r.running), "env_limit").map( - ({ bucket, sum, env }) => ({ - bucket, - value: env > 0 ? Math.round((sum / env) * 100) : 0, - }) - ); - const peak = points.reduce((max, p) => Math.max(max, p.value), 0); - return { points, total: peak, formatTotal: (v) => `${v}% peak` }; + const points = rows.map((r) => { + const limit = tileNumber(r.env_limit); + return { + bucket: tileTimeToMs(r.t), + value: limit > 0 ? Math.round((tileNumber(r.running) / limit) * 100) : 0, + }; + }); + return { points, total: peakOf(points), formatTotal: (v) => `${v}% peak` }; }, }, { id: "backlog", label: "Backlog", - description: "How many runs are waiting across these queues, over time.", + description: "How many runs are waiting across the environment, over time.", color: "var(--color-queues)", - query: `SELECT timeBucket() AS t,\n queue,\n max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t, queue\nORDER BY t`, + query: `SELECT timeBucket() AS t,\n max(max_env_queued) AS queued\nFROM env_metrics\nGROUP BY t\nORDER BY t`, derive: (rows) => { - const points = sumByBucket(rows, (r) => tileNumber(r.queued)).map(({ bucket, sum }) => ({ - bucket, - value: sum, + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.queued), })); - const peak = points.reduce((max, p) => Math.max(max, p.value), 0); - return { points, total: peak, formatTotal: (v) => `${v.toLocaleString()} peak` }; + return { points, total: peakOf(points), formatTotal: (v) => `${v.toLocaleString()} peak` }; }, }, { @@ -1343,14 +1303,15 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-queues)", label: "p95" }, { color: "var(--color-warning)", label: "Over 1 min" }, ], - // quantilesMerge over the set's rows in a bucket is the true p95 across the union of samples - // (merging quantile states is valid; averaging per-queue percentiles would not be). - query: `SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + query: `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`, formatValue: formatWaitMs, formatAxis: formatWaitMs, derive: (rows) => { - const points = rows.map((r) => ({ bucket: tileTimeToMs(r.t), value: tileNumber(r.p95) })); - const worst = points.reduce((max, p) => Math.max(max, p.value), 0); + const points = rows.map((r) => ({ + bucket: tileTimeToMs(r.t), + value: tileNumber(r.samples) > 0 ? tileNumber(r.p95) : null, + })); + const worst = peakOf(points); return { points, total: worst, @@ -1366,7 +1327,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ totalTooltip: "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], - query: `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`, + query: `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`, derive: (rows) => { const points = rows.map((r) => ({ bucket: tileTimeToMs(r.t), @@ -1376,7 +1337,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ // scales with poll rate and window length); the fraction of buckets with a throttle is. // The data path fills gaps (zero-fill for this counter), so every bucket in the window is // present and `points.length` is the honest denominator. - const nonzero = points.filter((p) => p.value > 0).length; + const nonzero = points.filter((p) => p.value !== null && p.value > 0).length; const pct = points.length > 0 ? Math.round((nonzero / points.length) * 100) : 0; return { points, @@ -1388,10 +1349,12 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ }, ]; -// When a search matches no queues the set is empty. We still fetch (hooks can't be conditional), -// but with a queue name that can't exist so the IN filter returns nothing and the tile falls -// through to its "No activity" empty state instead of silently widening to the whole environment. -const NO_QUEUES_SENTINEL = "__no_queues__"; +/** + * Bucket floor shared by every hero tile. Scheduling delay and throttling are event-driven, so at + * the 10-second width a short range would otherwise pick, most buckets hold no samples at all. One + * floor for all four keeps their x-axes identical, which the shared hover crosshair relies on. + */ +const HERO_CHART_MIN_BUCKET_SECONDS = 60; type TileTimeRange = MetricResourceTimeRange; @@ -1401,7 +1364,6 @@ type TileTimeRange = MetricResourceTimeRange; function QueueEnvMetricChart({ tile, timeRange, - queueNames, referenceLines, thresholdStroke, warningOverlay, @@ -1409,8 +1371,6 @@ function QueueEnvMetricChart({ }: { tile: QueueHeaderTile; timeRange: TileTimeRange; - /** The visible queue set (post-search, post-pagination) the chart scopes to. */ - queueNames: string[]; referenceLines?: Array<{ y: number; label?: string; @@ -1427,9 +1387,6 @@ function QueueEnvMetricChart({ const project = useProject(); const environment = useEnvironment(); - // Scope to exactly the queues the table is showing. Empty set => sentinel that matches nothing, - // so the tile shows "No activity" rather than the whole environment. The hook re-fetches when - // this list changes (it keys on the joined names), so search/pagination reflow the charts. const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { organizationId: organization.id, projectId: project.id, @@ -1437,7 +1394,7 @@ function QueueEnvMetricChart({ timeRange, defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, fillGaps: true, - queues: queueNames.length > 0 ? queueNames : [NO_QUEUES_SENTINEL], + minBucketSeconds: HERO_CHART_MIN_BUCKET_SECONDS, }); const { points, total, formatTotal, totalClassName } = tile.derive(rows); @@ -1461,7 +1418,7 @@ function QueueEnvMetricChart({ () => buildActivityTimeAxis(data), [data] ); - const hasData = data.length > 0 && data.some((p) => (p[tile.id] as number) > 0); + const hasData = data.length > 0 && data.some((p) => Number(p[tile.id] ?? 0) > 0); // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) diff --git a/apps/webapp/app/routes/resources.metric.tsx b/apps/webapp/app/routes/resources.metric.tsx index 1808f6107e4..af98a54b84d 100644 --- a/apps/webapp/app/routes/resources.metric.tsx +++ b/apps/webapp/app/routes/resources.metric.tsx @@ -52,6 +52,7 @@ const MetricWidgetQuery = z.object({ tags: z.array(z.string()).optional(), // Opt into server-side gap fill (carry-forward for gauges, zero-fill for counters). fillGaps: z.boolean().optional(), + minBucketSeconds: z.number().int().positive().max(86_400).optional(), userAuthoredQuery: z.boolean().optional(), }); @@ -89,6 +90,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { providers, tags: _tags, fillGaps, + minBucketSeconds, userAuthoredQuery, } = submission.data; @@ -128,6 +130,7 @@ export const action = async ({ request }: ActionFunctionArgs) => { operations, providers, fillGaps, + minBucketSeconds, userAuthoredQuery, // Set higher concurrency if many widgets are on screen at once customOrgConcurrencyLimit: env.METRIC_WIDGET_DEFAULT_ORG_CONCURRENCY_LIMIT, diff --git a/apps/webapp/app/services/queryService.server.ts b/apps/webapp/app/services/queryService.server.ts index bcceca25bbb..755da8ec3a3 100644 --- a/apps/webapp/app/services/queryService.server.ts +++ b/apps/webapp/app/services/queryService.server.ts @@ -9,8 +9,8 @@ import { import type { CustomerQuerySource } from "@trigger.dev/database"; import { calculateTimeBucketInterval, + intervalToSeconds, type TableSchema, - type TimeBucketInterval, type WhereClauseCondition, } from "@internal/tsql"; import { z } from "zod"; @@ -135,15 +135,6 @@ export function isQueryConcurrencyRejection(error: unknown): boolean { ); } -const INTERVAL_UNIT_SECONDS: Record = { - SECOND: 1, - MINUTE: 60, - HOUR: 3_600, - DAY: 86_400, - WEEK: 604_800, - MONTH: 2_592_000, -}; - function floorToSeconds(date: Date, alignSeconds: number): Date { const ms = alignSeconds * 1000; return new Date(Math.floor(date.getTime() / ms) * ms); @@ -167,16 +158,21 @@ function resolveQueryClientType(schema: TableSchema | undefined): ClientType { * rollup's granularity. The rollup has identical logical columns, so only the physical * table (and therefore rows read) changes. */ -function resolveRollup(schema: TableSchema, timeRange: { from: Date; to: Date }): TableSchema { +function resolveRollup( + schema: TableSchema, + timeRange: { from: Date; to: Date }, + minBucketSeconds?: number +): TableSchema { if (!schema.rollups || schema.rollups.length === 0) { return schema; } const interval = calculateTimeBucketInterval( timeRange.from, timeRange.to, - schema.timeBucketThresholds + schema.timeBucketThresholds, + minBucketSeconds ); - const intervalSeconds = interval.value * INTERVAL_UNIT_SECONDS[interval.unit]; + const intervalSeconds = intervalToSeconds(interval); const best = [...schema.rollups] .sort((a, b) => b.minIntervalSeconds - a.minIntervalSeconds) .find((r) => r.minIntervalSeconds <= intervalSeconds); @@ -374,7 +370,9 @@ export async function executeQuery( ); // Serve coarse-bucket queries from the table's rollup when one qualifies. const effectiveSchemas = matchedSchema?.rollups - ? querySchemas.map((s) => (s === matchedSchema ? resolveRollup(s, timeRange) : s)) + ? querySchemas.map((s) => + s === matchedSchema ? resolveRollup(s, timeRange, baseOptions.minBucketSeconds) : s + ) : querySchemas; const queryCacheSettings: ClickHouseSettings = matchedSchema?.queryCache diff --git a/apps/webapp/vite.config.ts b/apps/webapp/vite.config.ts index 0b33a4edb6f..4d7efbfc5ee 100644 --- a/apps/webapp/vite.config.ts +++ b/apps/webapp/vite.config.ts @@ -59,6 +59,8 @@ export default defineConfig({ "@kapaai/react-sdk", "@fingerprintjs/fingerprintjs-pro", "@fingerprintjs/fingerprintjs-pro-spa", + "recharts", + /^victory-vendor/, ], optimizeDeps: { include: ["cron-parser"], diff --git a/internal-packages/clickhouse/src/client/tsql.ts b/internal-packages/clickhouse/src/client/tsql.ts index 2cf586c36f1..f61009237e4 100644 --- a/internal-packages/clickhouse/src/client/tsql.ts +++ b/internal-packages/clickhouse/src/client/tsql.ts @@ -114,6 +114,11 @@ export interface ExecuteTSQLOptions { * (counters zero-fill, gauges carry forward). Off by default. */ fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width. + */ + minBucketSeconds?: number; /** * Set when `query` was written by whoever made the request rather than by us. * A rejection of their SQL is then their mistake, not a bug on our side. @@ -204,6 +209,7 @@ export async function executeTSQL( whereClauseFallback: options.whereClauseFallback, timeRange: options.timeRange, fillGaps: options.fillGaps, + minBucketSeconds: options.minBucketSeconds, }); generatedSql = sql; diff --git a/internal-packages/tsql/src/index.ts b/internal-packages/tsql/src/index.ts index 1ebd1a60a5d..a40ef63a62a 100644 --- a/internal-packages/tsql/src/index.ts +++ b/internal-packages/tsql/src/index.ts @@ -134,7 +134,9 @@ export { // Re-export time bucket utilities export { BUCKET_THRESHOLDS, + INTERVAL_UNIT_SECONDS, calculateTimeBucketInterval, + intervalToSeconds, type BucketThreshold, type TimeBucketInterval, } from "./query/time_buckets.js"; @@ -565,6 +567,12 @@ export interface CompileTSQLOptions { * Off by default; output is unchanged when not set. */ fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width (e.g. a + * percentile over buckets that often contain no samples at all). + */ + minBucketSeconds?: number; } /** @@ -624,6 +632,7 @@ export function compileTSQL(query: string, options: CompileTSQLOptions): PrintRe enforcedWhereClause, timeRange: options.timeRange, fillGaps: options.fillGaps, + minBucketSeconds: options.minBucketSeconds, }); // 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level) diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index ed59b9977fe..adfa62cedb0 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -624,7 +624,8 @@ export class ClickHousePrinter { const interval = calculateTimeBucketInterval( timeRange.from, timeRange.to, - tableSchema.timeBucketThresholds + tableSchema.timeBucketThresholds, + this.context.minBucketSeconds ); const bucketSql = `toStartOfInterval(${escapeClickHouseIdentifier(clickhouseColumnName)}, INTERVAL ${interval.value} ${interval.unit})`; @@ -3551,7 +3552,8 @@ export class ClickHousePrinter { const interval = calculateTimeBucketInterval( timeRange.from, timeRange.to, - tableSchema.timeBucketThresholds + tableSchema.timeBucketThresholds, + this.context.minBucketSeconds ); // Emit toStartOfInterval(column, INTERVAL N UNIT) diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts index a964e2e04af..3ee3da644d0 100644 --- a/internal-packages/tsql/src/query/printer_context.ts +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -128,6 +128,12 @@ export class PrinterContext { /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */ readonly fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width. + */ + readonly minBucketSeconds?: number; + constructor( /** Schema registry containing allowed tables and columns */ public readonly schema: SchemaRegistry, @@ -143,7 +149,9 @@ export class PrinterContext { /** Time range for timeBucket() interval calculation */ timeRange?: TimeRange, /** Opt-in gap-fill for time-bucketed queries */ - fillGaps?: boolean + fillGaps?: boolean, + /** Floor for the timeBucket() interval, in seconds */ + minBucketSeconds?: number ) { // Initialize with default settings this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings }; @@ -151,6 +159,7 @@ export class PrinterContext { this.enforcedWhereClause = enforcedWhereClause; this.timeRange = timeRange; this.fillGaps = fillGaps; + this.minBucketSeconds = minBucketSeconds; } /** @@ -286,6 +295,11 @@ export interface PrinterContextOptions { timeRange?: TimeRange; /** When true, time-bucketed queries emit rows for empty buckets (opt-in). */ fillGaps?: boolean; + /** + * Floor for the `timeBucket()` interval, in seconds. Widens buckets past what the range + * would pick, for series whose samples are too sparse to read at that width. + */ + minBucketSeconds?: number; } /** @@ -298,6 +312,7 @@ export function createPrinterContext(options: PrinterContextOptions): PrinterCon options.fieldMappings, options.enforcedWhereClause, options.timeRange, - options.fillGaps + options.fillGaps, + options.minBucketSeconds ); } diff --git a/internal-packages/tsql/src/query/time_buckets.test.ts b/internal-packages/tsql/src/query/time_buckets.test.ts index 451e74bccc1..c23fa8a09fd 100644 --- a/internal-packages/tsql/src/query/time_buckets.test.ts +++ b/internal-packages/tsql/src/query/time_buckets.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from "vitest"; -import { calculateTimeBucketInterval, type TimeBucketInterval } from "./time_buckets.js"; +import { + calculateTimeBucketInterval, + type BucketThreshold, + type TimeBucketInterval, +} from "./time_buckets.js"; /** * Helper to create a Date range from a start date and a duration @@ -178,4 +182,46 @@ describe("calculateTimeBucketInterval", () => { }); }); }); + + describe("minBucketSeconds floor", () => { + const tenSecondThresholds: BucketThreshold[] = [ + { maxRangeSeconds: 3 * 60 * 60, interval: { value: 10, unit: "SECOND" } }, + ]; + + it("should widen an interval below the floor", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds, 60) + ).toEqual({ value: 1, unit: "MINUTE" }); + }); + + it("should leave an interval already above the floor alone", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 5 * DAY); + expect(calculateTimeBucketInterval(from, to, undefined, 60)).toEqual({ + value: 6, + unit: "HOUR", + }); + }); + + it("should be a no-op when no floor is given", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds) + ).toEqual({ value: 10, unit: "SECOND" }); + }); + + it("should express the floor in the largest unit it divides evenly into", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds, 300) + ).toEqual({ value: 5, unit: "MINUTE" }); + }); + + it("should fall back to seconds for a floor that is not a whole number of minutes", () => { + const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR); + expect( + calculateTimeBucketInterval(from, to, tenSecondThresholds, 90) + ).toEqual({ value: 90, unit: "SECOND" }); + }); + }); }); diff --git a/internal-packages/tsql/src/query/time_buckets.ts b/internal-packages/tsql/src/query/time_buckets.ts index 04edde007d3..6fbbc1973a9 100644 --- a/internal-packages/tsql/src/query/time_buckets.ts +++ b/internal-packages/tsql/src/query/time_buckets.ts @@ -57,6 +57,36 @@ export const BUCKET_THRESHOLDS: BucketThreshold[] = [ /** Default interval for very large ranges (365+ days) */ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" }; +/** Seconds in each bucket unit. MONTH is nominal (30 days) and only used for comparisons. */ +export const INTERVAL_UNIT_SECONDS: Record = { + SECOND: 1, + MINUTE: 60, + HOUR: 3600, + DAY: 86400, + WEEK: 604800, + MONTH: 2592000, +}; + +/** Duration of a bucket interval, in seconds. */ +export function intervalToSeconds(interval: TimeBucketInterval): number { + return interval.value * INTERVAL_UNIT_SECONDS[interval.unit]; +} + +/** + * Express a duration in seconds as a bucket interval, preferring the largest unit it divides + * evenly into so the emitted `INTERVAL N UNIT` reads naturally (120 -> 2 MINUTE, not 120 SECOND). + */ +function secondsToInterval(seconds: number): TimeBucketInterval { + const units: Array = ["WEEK", "DAY", "HOUR", "MINUTE"]; + for (const unit of units) { + const unitSeconds = INTERVAL_UNIT_SECONDS[unit]; + if (seconds >= unitSeconds && seconds % unitSeconds === 0) { + return { value: seconds / unitSeconds, unit }; + } + } + return { value: Math.max(1, Math.round(seconds)), unit: "SECOND" }; +} + /** * Calculate the most appropriate time bucket interval for a given time range. * @@ -66,6 +96,9 @@ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" }; * * @param from - Start of the time range * @param to - End of the time range + * @param thresholds - Table-specific thresholds, defaulting to `BUCKET_THRESHOLDS` + * @param minBucketSeconds - Floor for the returned interval, for series whose samples are too + * sparse to be meaningful at the range's natural bucket width * @returns The recommended bucket interval * * @example @@ -86,10 +119,21 @@ const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" }; export function calculateTimeBucketInterval( from: Date, to: Date, - thresholds?: BucketThreshold[] + thresholds?: BucketThreshold[], + minBucketSeconds?: number ): TimeBucketInterval { const rangeSeconds = Math.abs(to.getTime() - from.getTime()) / 1000; + const interval = pickInterval(rangeSeconds, thresholds); + + if (minBucketSeconds !== undefined && intervalToSeconds(interval) < minBucketSeconds) { + return secondsToInterval(minBucketSeconds); + } + + return interval; +} + +function pickInterval(rangeSeconds: number, thresholds?: BucketThreshold[]): TimeBucketInterval { for (const threshold of thresholds ?? BUCKET_THRESHOLDS) { if (rangeSeconds < threshold.maxRangeSeconds) { return threshold.interval; From 2be18aaebe365b91f6862bb0f7cf309830dfadfe Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 12:12:02 +0100 Subject: [PATCH 2/8] fix(tsql): carry the bucket-width floor into child printer contexts createChildContext forwarded every other context-carried option, including fillGaps, but not minBucketSeconds, so a nested query part would have computed an unfloored bucket interval. No caller reaches this today, so it is latent rather than broken, but it is the same omission that would have been a live bug for fillGaps. --- internal-packages/tsql/src/query/printer_context.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal-packages/tsql/src/query/printer_context.ts b/internal-packages/tsql/src/query/printer_context.ts index 3ee3da644d0..69fb5b6e944 100644 --- a/internal-packages/tsql/src/query/printer_context.ts +++ b/internal-packages/tsql/src/query/printer_context.ts @@ -241,7 +241,8 @@ export class PrinterContext { this.fieldMappings, this.enforcedWhereClause, this.timeRange, - this.fillGaps + this.fillGaps, + this.minBucketSeconds ); // Share the same values map so parameters are unified child.values = this.values; From 97875c8bd1448029398e98a646bf57f3237b6c15 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 13:01:21 +0100 Subject: [PATCH 3/8] fix(webapp): keep the throttled readout independent of the chart's bucket width The Throttled tile's headline is a share of buckets that saw any throttling, not a peak, so the 60-second bucket floor inflated it: a single brief throttle now marked a whole minute instead of ten seconds. On the same seeded data it read 17% before the floor and 85% after, for identical throttle events. The chart keeps the floor, because a readable line was the point of it. The headline now comes from a second query at the range's natural bucket width, so it means what its tooltip says regardless of how the plotted buckets are sized. Tiles declare this via an optional `readout`; the other three measure peaks, which are width-invariant for a max over gauges, so they are unchanged and issue no extra query. An empty query is now a no-op in useMetricResourceQuery, so the hook can be called unconditionally for tiles that have no separate readout. --- .../app/hooks/useMetricResourceQuery.ts | 5 ++ .../route.tsx | 70 ++++++++++++++----- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/apps/webapp/app/hooks/useMetricResourceQuery.ts b/apps/webapp/app/hooks/useMetricResourceQuery.ts index 2ada75735bd..59376f984db 100644 --- a/apps/webapp/app/hooks/useMetricResourceQuery.ts +++ b/apps/webapp/app/hooks/useMetricResourceQuery.ts @@ -51,6 +51,7 @@ function cacheSet(key: string, rows: MetricResourceRow[]) { * back-navigation to the queues list) shows its last data immediately and revalidates in the * background rather than flashing a loading skeleton. */ +/** An empty query means the caller has nothing to ask for, so no request is made. */ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryOptions) { const { organizationId, @@ -101,6 +102,10 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO const loadedKeyRef = useRef(null); const load = useCallback(() => { + if (!query) { + setIsLoading(false); + return; + } abortRef.current?.abort(); const controller = new AbortController(); abortRef.current = controller; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index a1855ede8f6..c9794d8c618 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1228,6 +1228,19 @@ type QueueHeaderTile = { formatTotal?: (total: number) => string; totalClassName?: string; }; + /** + * Optional second query, run at the range's natural bucket width, that owns the headline readout. + * A readout measured in buckets rather than in values (a share of buckets, not a peak) would + * otherwise move whenever this tile's bucket floor widens the plotted buckets. + */ + readout?: { + query: string; + derive: (rows: MetricTileRow[]) => { + total: number; + formatTotal?: (total: number) => string; + totalClassName?: string; + }; + }; }; function tileNumber(value: number | string | null): number { @@ -1245,6 +1258,8 @@ function peakOf(points: TilePoint[]): number { return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } +const THROTTLED_QUERY = `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`; + const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { id: "saturation", @@ -1327,24 +1342,31 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ totalTooltip: "The share of the selected window with at least one blocked dequeue.", color: "var(--color-queues)", legend: [{ color: "var(--color-warning)", label: "Throttled" }], - query: `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + query: THROTTLED_QUERY, derive: (rows) => { const points = rows.map((r) => ({ bucket: tileTimeToMs(r.t), value: tileNumber(r.throttled), })); - // Share of the window that saw any throttling. A raw event sum isn't interpretable (it - // scales with poll rate and window length); the fraction of buckets with a throttle is. - // The data path fills gaps (zero-fill for this counter), so every bucket in the window is - // present and `points.length` is the honest denominator. - const nonzero = points.filter((p) => p.value !== null && p.value > 0).length; - const pct = points.length > 0 ? Math.round((nonzero / points.length) * 100) : 0; - return { - points, - total: pct, - formatTotal: (v) => `${v}% of current period`, - totalClassName: pct > 0 ? "text-warning" : undefined, - }; + return { points, total: peakOf(points) }; + }, + readout: { + query: THROTTLED_QUERY, + /** + * Share of the window that saw any throttling. A raw event sum isn't interpretable (it + * scales with poll rate and window length); the fraction of buckets with a throttle is. + * Gap fill zero-fills this counter, so every bucket in the window is present and the row + * count is the honest denominator. + */ + derive: (rows) => { + const nonzero = rows.filter((r) => tileNumber(r.throttled) > 0).length; + const pct = rows.length > 0 ? Math.round((nonzero / rows.length) * 100) : 0; + return { + total: pct, + formatTotal: (v) => `${v}% of current period`, + totalClassName: pct > 0 ? "text-warning" : undefined, + }; + }, }, }, ]; @@ -1387,17 +1409,27 @@ function QueueEnvMetricChart({ const project = useProject(); const environment = useEnvironment(); - const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { + const sharedOptions = { organizationId: organization.id, projectId: project.id, environmentId: environment.id, timeRange, defaultPeriod: QUEUE_METRICS_DEFAULT_PERIOD, fillGaps: true, + }; + + const { rows, showLoading, failed } = useMetricResourceQuery(tile.query, { + ...sharedOptions, minBucketSeconds: HERO_CHART_MIN_BUCKET_SECONDS, }); - const { points, total, formatTotal, totalClassName } = tile.derive(rows); + const readoutResult = useMetricResourceQuery(tile.readout?.query ?? "", sharedOptions); + + const derived = tile.derive(rows); + const points = derived.points; + const { total, formatTotal, totalClassName } = tile.readout + ? tile.readout.derive(readoutResult.rows) + : derived; // Same point shape the shared axis/tooltip helpers expect. const data = points @@ -1423,9 +1455,11 @@ function QueueEnvMetricChart({ // Peak readout lives in the card title (ChartCard has no dedicated value slot). A zero/empty // total renders no readout at all (skipping "0% peak", "0 peak", "0" and the p95 "–" placeholder) // so the card title stands alone until there's a non-zero value to show. - const peak = showLoading ? ( + const readoutLoading = tile.readout ? readoutResult.showLoading : showLoading; + const readoutFailed = tile.readout ? readoutResult.failed : failed; + const peak = readoutLoading ? ( - ) : failed || total === 0 ? null : formatTotal ? ( + ) : readoutFailed || total === 0 ? null : formatTotal ? ( formatTotal(total) ) : ( total.toLocaleString() @@ -1445,7 +1479,7 @@ function QueueEnvMetricChart({ /> {peak != null ? ( - tile.totalTooltip && !showLoading ? ( + tile.totalTooltip && !readoutLoading ? ( Date: Mon, 3 Aug 2026 13:31:14 +0100 Subject: [PATCH 4/8] fix(webapp): apply the sparse-series bucket floor to the queue detail charts The queue detail page has the same event-driven series as the Queues list hero row, and the same problem: scheduling delay and throttling only have samples when something started or was held back, so at the 10-second width a short range picks, most buckets held nothing and the lines read as a run of zeros. Scheduling delay (p50/p95/p99), Throttled, and the per-key mean delay on the Concurrency keys tab now take the same 60-second floor, and the two delay charts break where a bucket genuinely has no samples instead of drawing a zero. The gauge charts on this page (concurrency, queue depth, keys with backlog, worst key wait) carry forward and read correctly at any width, so they are left alone. None of this page's charts carry a headline readout, so there is no share-of- buckets figure here to skew the way the list page's throttled readout did. The floor and the no-samples break are plumbed through the shared queue-metric card, so the task detail page and run inspector can opt in later without further changes. --- .../components/queues/QueueMetricCards.tsx | 23 +++++++++++++++---- .../route.tsx | 18 +++++++++++++-- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index 95dabff0d71..b45759382c3 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -54,6 +54,8 @@ export function useQueueMetric( defaultPeriod?: string; /** Poll ClickHouse on this cadence (ms). Omit to use the query's default interval. */ refreshIntervalMs?: number; + /** Floor for the bucket width, for series too sparse to read at the range's natural width. */ + minBucketSeconds?: number; } ) { return useMetricResourceQuery(query, { @@ -62,6 +64,7 @@ export function useQueueMetric( defaultPeriod: opts.defaultPeriod ?? QUEUE_METRICS_DEFAULT_PERIOD, queues: [opts.queueName], fillGaps: opts.fillGaps, + minBucketSeconds: opts.minBucketSeconds, refreshIntervalMs: opts.refreshIntervalMs, }); } @@ -120,6 +123,14 @@ type QueueMetricChartProps = { /** Reports whether the chart has data to plot (false once it settles on the "no activity" state), * so a wrapping card can hide the legend to match. */ onHasDataChange?: (hasData: boolean) => void; + /** Floor for the bucket width, for series too sparse to read at the range's natural width. */ + minBucketSeconds?: number; + /** + * Column whose value counts the samples behind the plotted series. Where it is zero the metric + * has nothing to report, so every series breaks there instead of reading as a real zero. Keep it + * out of `series` — it is read for this test only, never drawn. + */ + sampleCountColumn?: string; }; // Bare chart (no card chrome) so it can live inside a shared card, e.g. a tabbed panel. @@ -136,6 +147,8 @@ export function QueueMetricChart({ carryBackfill, thresholdStroke, onHasDataChange, + minBucketSeconds, + sampleCountColumn, }: QueueMetricChartProps) { const { rows, showLoading, failed } = useQueueMetric(query, { ids, @@ -143,15 +156,17 @@ export function QueueMetricChart({ queueName, fillGaps, defaultPeriod, + minBucketSeconds, }); const data = useMemo(() => { const points = rows .map((r) => { - const point: { bucket: number } & Record = { + const point: { bucket: number } & Record = { bucket: clickhouseTimeToMs(r.t), }; - for (const s of series) point[s.key] = toNumber(r[s.key]); + const hasSamples = sampleCountColumn ? toNumber(r[sampleCountColumn]) > 0 : true; + for (const s of series) point[s.key] = hasSamples ? toNumber(r[s.key]) : null; return point; }) .filter((p) => Number.isFinite(p.bucket)); @@ -160,7 +175,7 @@ export function QueueMetricChart({ // value and carry it back over the earlier buckets so the line doesn't start at a false 0. if (carryBackfill?.length) { for (const key of carryBackfill) { - const first = points.findIndex((p) => p[key] > 0); + const first = points.findIndex((p) => toNumber(p[key]) > 0); if (first > 0) { const value = points[first]![key]!; for (let i = 0; i < first; i++) points[i]![key] = value; @@ -168,7 +183,7 @@ export function QueueMetricChart({ } } return points; - }, [rows, series, carryBackfill]); + }, [rows, series, carryBackfill, sampleCountColumn]); const chartConfig = useMemo(() => { const cfg: ChartConfig = {}; diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx index 37f955cd9d5..c93644ff4da 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues_.$queueParam/route.tsx @@ -191,6 +191,14 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const CK_LIVE_LIMIT = 50; +/** + * Bucket floor for this page's event-driven charts (scheduling delay, throttling). Their samples + * only exist when something started or was held back, so at the 10-second width a short range + * picks, most buckets hold nothing and the line reads as a run of zeros. Gauge charts on this page + * (concurrency, queue depth, backlogged keys) carry forward and are left at the natural width. + */ +const SPARSE_CHART_MIN_BUCKET_SECONDS = 60; + // Whole-queue oldest wait right now: for keyed queues the per-key breakdown carries the oldest // enqueue time per key, so the queue's oldest is the max wait across keys; otherwise fall back to // the queue's oldest message directly. Returns null when nothing is waiting. @@ -470,8 +478,10 @@ function OverviewCharts({ info="How long runs wait before they start." showLegend className="aspect-[2/1]" - query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SPARSE_CHART_MIN_BUCKET_SECONDS} + sampleCountColumn="samples" ids={ids} timeRange={timeRange} queueName={queueName} @@ -493,6 +503,7 @@ function OverviewCharts({ className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SPARSE_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -1003,7 +1014,10 @@ function KeyDrilldown({ 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SPARSE_CHART_MIN_BUCKET_SECONDS} + sampleCountColumn="samples" ids={ids} timeRange={timeRange} queueName={queueName} From 6208a8b55fcdeec004f4f8e0bb7441bc9ac1377f Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 13:50:14 +0100 Subject: [PATCH 5/8] fix(webapp): give every chart in a synced group the same bucket width Flooring only the sparse charts on the queue detail page broke the shared hover crosshair. It is drawn as a recharts ReferenceLine on a category x-axis, so it only appears where the hovered bucket timestamp exists in the other chart's own data: a 10-second timestamp has no match in a 60-second series, so hovering Concurrency drew nothing on Scheduling delay or Throttled. Measured before the fix, hovering Concurrency reached 2 of the 4 other charts; now it reaches 4. Every chart inside each ChartSyncProvider on the page therefore takes the same floor, which is what the Queues list hero row already does. The gauges lose some resolution (a max over a wider bucket is still the same kind of value) in exchange for the crosshair working across the row. Also: a chart whose every plotted value is null still had rows, so it reported itself as having data while rendering its own no-data placeholder, leaving the series legend stranded above it. Presence is now derived from the plotted values rather than the row count. --- .../components/queues/QueueMetricCards.tsx | 9 +++++-- .../route.tsx | 24 ++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/apps/webapp/app/components/queues/QueueMetricCards.tsx b/apps/webapp/app/components/queues/QueueMetricCards.tsx index b45759382c3..65865205c55 100644 --- a/apps/webapp/app/components/queues/QueueMetricCards.tsx +++ b/apps/webapp/app/components/queues/QueueMetricCards.tsx @@ -220,9 +220,14 @@ export function QueueMetricChart({ // Report data presence so a wrapping card can hide its legend when the chart settles on the // "no activity" state. Only report once loaded, so the legend stays put while loading. + const hasPlottedData = useMemo( + () => data.some((point) => series.some((s) => point[s.key] != null)), + [data, series] + ); + useEffect(() => { - if (!showLoading) onHasDataChange?.(!failed && data.length > 0); - }, [showLoading, failed, data.length, onHasDataChange]); + if (!showLoading) onHasDataChange?.(!failed && hasPlottedData); + }, [showLoading, failed, hasPlottedData, onHasDataChange]); return ( { const CK_LIVE_LIMIT = 50; /** - * Bucket floor for this page's event-driven charts (scheduling delay, throttling). Their samples - * only exist when something started or was held back, so at the 10-second width a short range - * picks, most buckets hold nothing and the line reads as a run of zeros. Gauge charts on this page - * (concurrency, queue depth, backlogged keys) carry forward and are left at the natural width. + * Bucket floor for the charts in a synced group. The event-driven series (scheduling delay, + * throttling) need it: their samples only exist when something started or was held back, so at the + * 10-second width a short range picks, most buckets hold nothing and the line reads as a run of + * zeros. The gauges beside them take the same floor because the shared hover crosshair is a + * recharts ReferenceLine on a category x-axis, so it only draws where the hovered bucket + * timestamp exists in the other chart's own data — mixing widths in one group silently drops it. */ -const SPARSE_CHART_MIN_BUCKET_SECONDS = 60; +const SYNCED_CHART_MIN_BUCKET_SECONDS = 60; // Whole-queue oldest wait right now: for keyed queues the per-key breakdown carries the oldest // enqueue time per key, so the queue's oldest is the max wait across keys; otherwise fall back to @@ -417,6 +419,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, max(max_running) AS running, max(max_limit) AS limit\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -443,6 +446,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, max(max_queued) AS queued\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -462,6 +466,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t,\n deltaSumTimestampMerge(enqueue_delta) AS enqueued,\n deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -480,7 +485,7 @@ function OverviewCharts({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[1]) AS p50,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n round(quantilesMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[4]) AS p99,\n sum(wait_ms_count) AS samples\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps - minBucketSeconds={SPARSE_CHART_MIN_BUCKET_SECONDS} + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" ids={ids} timeRange={timeRange} @@ -503,7 +508,7 @@ function OverviewCharts({ className="aspect-[2/1] sm:col-span-2 sm:aspect-[4/1]" query={`SELECT timeBucket() AS t, sum(throttled_count) AS throttled\nFROM queue_metrics\nGROUP BY t\nORDER BY t`} fillGaps - minBucketSeconds={SPARSE_CHART_MIN_BUCKET_SECONDS} + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -993,6 +998,7 @@ function KeyDrilldown({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, max(max_queued) AS queued, max(max_running) AS running\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -1006,6 +1012,8 @@ function KeyDrilldown({ title={`Key ${keyName}: throughput`} className="aspect-[2/1]" query={`SELECT timeBucket() AS t, deltaSumTimestampMerge(started_delta) AS started\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} + fillGaps + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} ids={ids} timeRange={timeRange} queueName={queueName} @@ -1016,7 +1024,7 @@ function KeyDrilldown({ className="aspect-[2/1]" query={`SELECT timeBucket() AS t, if(sum(wait_ms_count) > 0, round(sum(wait_ms_sum) / sum(wait_ms_count)), 0) AS wait, sum(wait_ms_count) AS samples\nFROM queue_metrics_by_key\nWHERE ${pin}\nGROUP BY t\nORDER BY t`} fillGaps - minBucketSeconds={SPARSE_CHART_MIN_BUCKET_SECONDS} + minBucketSeconds={SYNCED_CHART_MIN_BUCKET_SECONDS} sampleCountColumn="samples" ids={ids} timeRange={timeRange} From 3c34f056ca9395e8806c81ec22fb731152553b1e Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 14:07:11 +0100 Subject: [PATCH 6/8] fix(webapp): read the worst scheduling delay at the natural bucket width I justified giving only the Throttled tile a separate readout on the grounds that the other three measure peaks and a max over gauges is width-invariant. That is true of saturation and backlog, whose max of maxes is the same at any width, and false of the p95 tile: merging quantile states over a wider bucket yields a p95 between the sub-buckets' own, so the worst p95 over the plotted 60-second buckets can only be lower than the worst at the natural width, while the tooltip claims it is the worst in the window. Demonstrated in ClickHouse: two 240s samples among twenty in one 10-second sub-bucket give a worst-of-six-sub-buckets p95 of 240,000ms and a merged 60-second p95 of 5,000ms, a 48x understatement. The p95 tile now takes the same readout escape hatch as Throttled, so its headline is measured at the natural width while the line keeps the floor that makes it readable. Saturation and backlog stay as they were and issue no extra query. The plotted line is still a smoothed view, so a sub-minute spike above the warning threshold can lose its warning colour on the chart even though the headline reports it and colours itself. --- .../route.tsx | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index c9794d8c618..081bbb3ba01 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1258,6 +1258,8 @@ function peakOf(points: TilePoint[]): number { return points.reduce((max, p) => (p.value === null ? max : Math.max(max, p.value)), 0); } +const SCHEDULING_DELAY_QUERY = `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`; + const THROTTLED_QUERY = `SELECT timeBucket() AS t,\n sum(throttled_count) AS throttled\nFROM env_metrics\nGROUP BY t\nORDER BY t`; const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ @@ -1318,7 +1320,7 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ { color: "var(--color-queues)", label: "p95" }, { color: "var(--color-warning)", label: "Over 1 min" }, ], - query: `SELECT timeBucket() AS t,\n round(quantilesTDigestMerge(0.5, 0.9, 0.95, 0.99)(wait_quantiles)[3]) AS p95,\n sum(wait_ms_count) AS samples\nFROM env_metrics\nGROUP BY t\nORDER BY t`, + query: SCHEDULING_DELAY_QUERY, formatValue: formatWaitMs, formatAxis: formatWaitMs, derive: (rows) => { @@ -1326,13 +1328,27 @@ const QUEUE_HEADER_TILES: QueueHeaderTile[] = [ bucket: tileTimeToMs(r.t), value: tileNumber(r.samples) > 0 ? tileNumber(r.p95) : null, })); - const worst = peakOf(points); - return { - points, - total: worst, - formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), - totalClassName: worst >= 60_000 ? "text-warning" : undefined, - }; + return { points, total: peakOf(points) }; + }, + readout: { + query: SCHEDULING_DELAY_QUERY, + /** + * Merging quantile states over a wider bucket yields a p95 between the sub-buckets' own, so + * the worst p95 has to be read at the range's natural width or a burst of slow starts shorter + * than the plotted bucket is averaged away. Unlike the gauges, whose max of maxes is the same + * at any width. + */ + derive: (rows) => { + const worst = rows.reduce( + (max, r) => (tileNumber(r.samples) > 0 ? Math.max(max, tileNumber(r.p95)) : max), + 0 + ); + return { + total: worst, + formatTotal: (v) => (v > 0 ? formatWaitMs(v) : "–"), + totalClassName: worst >= 60_000 ? "text-warning" : undefined, + }; + }, }, }, { From 6309b12e531e4c2484d76e9436413d976b0911d4 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 14:22:13 +0100 Subject: [PATCH 7/8] perf(webapp): only fetch a tile readout while the bucket floor is binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delay and throttled readouts each issued their own request even when the range's natural bucket width was already at or above the 60-second floor, where the unfloored query compiles to the same SQL and returns the same rows — so two identical queries ran per tile, and both re-ran on every poll and focus. The decision now comes from the plotted spacing rather than from a copy of the table's bucket thresholds: spacing wider than the floor means the range picked it, so the chart's own rows are what the unfloored query would have returned and they are reused. Verified at both ends — a 1-hour range still issues both readouts, a 7-day range issues neither and the headlines are unchanged. That leaves the extra query only on the short ranges where the floor actually changes the buckets, which are also the cheapest ranges to scan. --- .../route.tsx | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx index 081bbb3ba01..0430058a7af 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.queues/route.tsx @@ -1230,8 +1230,11 @@ type QueueHeaderTile = { }; /** * Optional second query, run at the range's natural bucket width, that owns the headline readout. - * A readout measured in buckets rather than in values (a share of buckets, not a peak) would - * otherwise move whenever this tile's bucket floor widens the plotted buckets. + * For a headline that is not invariant to bucket width — a share of buckets, or a percentile, + * as opposed to a max over gauges — deriving it from the plotted rows would move the number + * whenever this tile's floor widens them. Only requested while the floor is actually widening + * anything; on ranges whose natural width is already at or above the floor the chart's own rows + * are identical and are reused. */ readout?: { query: string; @@ -1439,12 +1442,17 @@ function QueueEnvMetricChart({ minBucketSeconds: HERO_CHART_MIN_BUCKET_SECONDS, }); - const readoutResult = useMetricResourceQuery(tile.readout?.query ?? "", sharedOptions); - const derived = tile.derive(rows); const points = derived.points; + + const plottedBucketMs = points.length > 1 ? points[1]!.bucket - points[0]!.bucket : 0; + const floorWidenedBuckets = + plottedBucketMs > 0 && plottedBucketMs <= HERO_CHART_MIN_BUCKET_SECONDS * 1000; + const readoutQuery = tile.readout && floorWidenedBuckets ? tile.readout.query : ""; + const readoutResult = useMetricResourceQuery(readoutQuery, sharedOptions); + const { total, formatTotal, totalClassName } = tile.readout - ? tile.readout.derive(readoutResult.rows) + ? tile.readout.derive(readoutQuery ? readoutResult.rows : rows) : derived; // Same point shape the shared axis/tooltip helpers expect. From 8ad0c5902e5160503d18725249b2391d2c3fb371 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Mon, 3 Aug 2026 14:36:00 +0100 Subject: [PATCH 8/8] fix(webapp): drop a skipped query's stale rows and failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty-query short circuit returned before the block that resets rows and the failure flag on a new query signature, so a caller that stopped asking kept the last answer. On the Queues page that meant a readout which failed once on a short range stayed hidden after widening to a range where the readout is skipped entirely and the headline comes from the chart's own successful rows — the number would have been missing for the rest of the session. Reproduced by failing only the p95 readout on a 1-hour range (headline gone as designed) and then widening to 7 days: the headline now returns at 58.3s. Clearing rows as well as the flag, since no query means no data, and leaving another query's rows behind is the same trap the reset below guards against. --- apps/webapp/app/hooks/useMetricResourceQuery.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/hooks/useMetricResourceQuery.ts b/apps/webapp/app/hooks/useMetricResourceQuery.ts index 59376f984db..d219ee73c08 100644 --- a/apps/webapp/app/hooks/useMetricResourceQuery.ts +++ b/apps/webapp/app/hooks/useMetricResourceQuery.ts @@ -51,7 +51,11 @@ function cacheSet(key: string, rows: MetricResourceRow[]) { * back-navigation to the queues list) shows its last data immediately and revalidates in the * background rather than flashing a loading skeleton. */ -/** An empty query means the caller has nothing to ask for, so no request is made. */ +/** + * An empty query means the caller has nothing to ask for, so no request is made and any rows or + * failure left by a previous query are dropped — a caller that stops asking must not keep reading + * the last answer, or a stale failure would outlive the query that caused it. + */ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryOptions) { const { organizationId, @@ -103,6 +107,10 @@ export function useMetricResourceQuery(query: string, opts: MetricResourceQueryO const load = useCallback(() => { if (!query) { + abortRef.current?.abort(); + loadedKeyRef.current = cacheKey; + setRows(null); + setFailed(false); setIsLoading(false); return; }