feat(client): scheduler requests stats after first paint, with pending, not computed and error states (rows-first c4)

packages/buckaroo-js-core/pw-tests/server.spec.ts

@@ -485,6 +485,158 @@ test.describe('WebSocket data flow', () => {
     expect(await getCellText(page, COL.name, 0)).toBe('Alice');
   });
 
+  test('standalone page advertises stats_update on its WebSocket URL', async ({ page, request }) => {
+    const session = `ws-caps-${Date.now()}`;
+    await loadSession(request, session, csvPath);
+
+    // The server reads capabilities from ?caps= at open, before the client
+    // sends anything, so the page has to put them on the URL it connects to.
+    const socketUrls: string[] = [];
+    page.on('websocket', (socket) => socketUrls.push(socket.url()));
+    await page.goto(`${BASE}/s/${session}`);
+    await waitForGrid(page);
+
+    const wsUrls = socketUrls.filter((u) => u.includes(`/ws/${session}`));
+    expect(wsUrls).toHaveLength(1);
+    expect(new URL(wsUrls[0]).searchParams.get('caps')).toBe('stats_update');
+  });
+
+  // Rows-first c4: the standalone page's scheduler. No session the server can
+  // build here defers its stats (that needs a xorq expression), so these
+  // tests put the pending state on a real session's first frame and answer
+  // the page's stats_request themselves. Rows, config and the stats payload
+  // all come from the real server.
+  async function loadBuckarooSession(request: any, sessionId: string) {
+    const resp = await request.post(`${BASE}/load`, {
+      data: { session: sessionId, path: csvPath, mode: 'buckaroo' },
+    });
+    expect(resp.ok()).toBe(true);
+  }
+
+  test('a pending session: the page asks for stats after the first rows, merges the reply and shows them', async ({ page, request }) => {
+    const session = `ws-sched-${Date.now()}`;
+    await loadBuckarooSession(request, session);
+
+    const order: string[] = [];
+    const requests: any[] = [];
+    let realStats: unknown;
+    await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => {
+      const server = ws.connectToServer();
+      server.onMessage((message) => {
+        if (typeof message !== 'string') {
+          ws.send(message);
+          return;
+        }
+        const msg = JSON.parse(message);
+        if (msg.type === 'infinite_resp') order.push('infinite_resp');
+        if (msg.type === 'initial_state' && realStats === undefined) {
+          // The first frame as a deferring server sends it: no stats yet.
+          realStats = msg.df_data_dict.all_stats;
+          msg.df_data_dict.all_stats = [];
+          msg.df_meta = { ...msg.df_meta, stats: { status: 'pending', tier: 'schema', gen: 1 } };
+          ws.send(JSON.stringify(msg));
+          return;
+        }
+        ws.send(message);
+      });
+      ws.onMessage((message) => {
+        if (typeof message === 'string') {
+          const msg = JSON.parse(message);
+          if (msg.type === 'stats_request') {
+            order.push('stats_request');
+            requests.push(msg);
+            ws.send(JSON.stringify({
+              type: 'stats_update', stats_gen: msg.stats_gen, scope: msg.scope, tier: 'full',
+              final: true, payload: realStats, elapsed_ms: 1,
+            }));
+            return;
+          }
+        }
+        server.send(message);
+      });
+    });
+
+    await page.goto(`${BASE}/s/${session}`);
+    await waitForGrid(page);
+
+    // The stats arrive: the status bar says so and the pinned dtype row has its values.
+    await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'complete', { timeout: 10_000 });
+    await expect(page.locator('.ag-floating-top [col-id="b"]').first()).not.toHaveText('', { timeout: 10_000 });
+
+    // One whole-run request, for the gen on the first frame, after rows had arrived.
+    expect(requests).toEqual([{ type: 'stats_request', stats_gen: 1, scope: 'raw' }]);
+    expect(order.indexOf('infinite_resp')).toBeGreaterThanOrEqual(0);
+    expect(order.indexOf('infinite_resp')).toBeLessThan(order.indexOf('stats_request'));
+  });
+
+  test('a session whose stats are not computed: the page asks for nothing until the Compute summary stats button is clicked', async ({ page, request }) => {
+    const session = `ws-notcomputed-${Date.now()}`;
+    await loadBuckarooSession(request, session);
+
+    const requests: any[] = [];
+    await page.routeWebSocket(new RegExp(`/ws/${session}`), (ws) => {
+      const server = ws.connectToServer();
+      let firstFrame = true;
+      server.onMessage((message) => {
+        if (typeof message === 'string') {
+          const msg = JSON.parse(message);
+          if (msg.type === 'initial_state' && firstFrame) {
+            firstFrame = false;
+            msg.df_data_dict.all_stats = [];
+            msg.df_meta = { ...msg.df_meta, stats: { status: 'not_computed', tier: 'schema', gen: 4, reason: 'host' } };
+            ws.send(JSON.stringify(msg));
+            return;
+          }
+        }
+        ws.send(message);
+      });
+      ws.onMessage((message) => {
+        if (typeof message === 'string') {
+          const msg = JSON.parse(message);
+          if (msg.type === 'stats_request') {
+            requests.push(msg);
+            return;
+          }
+        }
+        server.send(message);
+      });
+    });
+
+    await page.goto(`${BASE}/s/${session}`);
+    await waitForGrid(page);
+    await expect(page.getByTestId('stats-status')).toHaveAttribute('data-stats-status', 'not_computed');
+    // The pinned rows are omitted, not left as placeholders.
+    await expect(page.locator('.ag-floating-top .ag-row')).toHaveCount(0);
+
+    // Nothing is asked for on its own, however long the page waits.
+    await page.waitForTimeout(2500);
+    expect(requests).toEqual([]);
+
+    await page.getByRole('button', { name: 'Compute summary stats' }).click();
+    await expect.poll(() => requests).toEqual([{ type: 'stats_request', stats_gen: 4, scope: 'raw', force: true }]);
+  });
+
+  test('a session that does not report df_meta.stats never gets a stats_request', async ({ page, request }) => {
+    const session = `ws-nosched-${Date.now()}`;
+    await loadBuckarooSession(request, session);
+
+    const sentTypes: string[] = [];
+    page.on('websocket', (socket) => {
+      socket.on('framesent', (frame) => {
+        if (typeof frame.payload === 'string') sentTypes.push(JSON.parse(frame.payload).type);
+      });
+    });
+    await page.goto(`${BASE}/s/${session}`);
+    await waitForGrid(page);
+    await expect(page.locator('.status-bar')).toBeVisible();
+    // Longer than the scheduler would wait for rows that never come.
+    await page.waitForTimeout(3000);
+
+    expect(sentTypes).toContain('infinite_request');
+    expect(sentTypes).not.toContain('stats_request');
+    await expect(page.getByTestId('stats-status')).toHaveCount(0);
+  });
+
   test('WebSocket receives data for scrolled rows', async ({ page, request }) => {
     // Create a larger dataset (100 rows) to force infinite scrolling
     const rows = [];

packages/buckaroo-js-core/pw-tests/stats-pending-pinned-rows.spec.ts

@@ -0,0 +1,50 @@
+/**
+ * Playwright test for the client states of the two-message protocol
+ * (rows-first c0a).
+ *
+ * While summary stats have not arrived, `df_meta.stats.status` decides what
+ * the pinned area shows. The StatsPendingPinnedRows story flips the status and
+ * supplies the stats on "complete":
+ *   - pending: every valueless pinned key shows a placeholder row with its own
+ *     row id
+ *   - not_computed: valueless pinned keys are omitted
+ *   - complete: the values appear and the color-mapped column restyles
+ */
+import { test, expect } from "@playwright/test";
+import { waitForCells } from "./ag-pw-utils";
+
+const STORY_URL =
+  "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-dfviewer-statspendingpinnedrows--primary&globals=&args=";
+
+test("pinned rows follow df_meta.stats.status: placeholders, omitted, then values and colors", async ({ page }) => {
+  const pageErrors: string[] = [];
+  page.on("pageerror", (e) => pageErrors.push(e.message));
+
+  await page.goto(STORY_URL);
+  await waitForCells(page);
+
+  const pinnedRows = page.locator(".ag-floating-top .ag-row");
+  const distinctPinnedRowIds = async () => {
+    const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id")));
+    return Array.from(new Set(ids)).sort();
+  };
+  const bodyCellA = page.locator('.ag-center-cols-container .ag-row[row-index="0"] [col-id="a"]');
+  const backgroundOf = (loc: typeof bodyCellA) => loc.evaluate((el) => getComputedStyle(el).backgroundColor);
+
+  // pending: one placeholder per pinned key, each with its own row id.
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]);
+  await expect(bodyCellA).toHaveText("1");
+  const backgroundBeforeBins = await backgroundOf(bodyCellA);
+
+  // not_computed: the valueless keys are omitted.
+  await page.getByTestId("status-not_computed").click();
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]);
+
+  // complete: the values appear, and the color-mapped column restyles now
+  // that its histogram bins exist.
+  await page.getByTestId("status-complete").click();
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]);
+  await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64");
+  await expect.poll(() => backgroundOf(bodyCellA), { timeout: 10_000 }).not.toBe(backgroundBeforeBins);
+  expect(pageErrors).toEqual([]);
+});

packages/buckaroo-js-core/pw-tests/stats-scheduler-states.spec.ts

@@ -0,0 +1,109 @@
+/**
+ * Playwright test for the client states of the stats wire (rows-first c4).
+ *
+ * The StatsSchedulerStates story flips df_meta.stats.status. Each status has
+ * its own text in the status bar's stats column, and the pinned area follows it
+ * as in c0a. Switching between statuses moves nothing above or beside the grid:
+ * the status bar keeps its box, and the grid keeps its position and width. The
+ * grid's height changes only with the pinned area: placeholders hold the height
+ * of the values that replace them, and omitted rows take theirs away. In
+ * "not_computed" the status bar offers a Compute summary stats button, which
+ * sends stats_request {force: true}.
+ */
+import { test, expect, Page } from "@playwright/test";
+import { waitForCells } from "./ag-pw-utils";
+
+const STORY_URL =
+  "http://localhost:6006/iframe.html?viewMode=story&id=buckaroo-statsschedulerstates--primary&globals=&args=";
+
+const boxes = async (page: Page) => {
+  const round = (b: { x: number; y: number; width: number; height: number } | null) =>
+    b && { x: Math.round(b.x), y: Math.round(b.y), width: Math.round(b.width), height: Math.round(b.height) };
+  return {
+    statusBar: round(await page.locator(".status-bar").boundingBox()),
+    grid: round(await page.locator(".df-viewer").boundingBox()),
+    widget: round(await page.locator(".buckaroo-widget").boundingBox()),
+  };
+};
+
+test("each stats status shows its own text in the status bar and nothing moves between them", async ({ page }) => {
+  const pageErrors: string[] = [];
+  page.on("pageerror", (e) => pageErrors.push(e.message));
+
+  await page.goto(STORY_URL);
+  await waitForCells(page);
+
+  const statsCell = page.getByTestId("stats-status");
+  const pinnedRows = page.locator(".ag-floating-top .ag-row");
+  const distinctPinnedRowIds = async () => {
+    const ids = await pinnedRows.evaluateAll((els) => els.map((e) => e.getAttribute("row-id")));
+    return Array.from(new Set(ids)).sort();
+  };
+
+  // pending: the loading text, and a placeholder for each pinned key.
+  await expect(statsCell).toHaveAttribute("data-stats-status", "pending");
+  await expect(statsCell).toContainText("Computing summary stats");
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]);
+  const reference = await boxes(page);
+  expect(reference.statusBar).not.toBeNull();
+  expect(reference.grid).not.toBeNull();
+  // Nothing above or beside the grid moves, whatever the status.
+  const staysPut = async () => {
+    const now = await boxes(page);
+    expect(now.statusBar).toEqual(reference.statusBar);
+    expect(now.grid).toMatchObject({ x: reference.grid!.x, y: reference.grid!.y, width: reference.grid!.width });
+    return now;
+  };
+
+  // not_computed: the control, and no pinned rows.
+  await page.getByTestId("status-not_computed").click();
+  await expect(statsCell).toHaveAttribute("data-stats-status", "not_computed");
+  await expect(page.getByRole("button", { name: "Compute summary stats" })).toBeVisible();
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]);
+  const withoutPinned = await staysPut();
+  // The omitted rows take their height with them.
+  expect(withoutPinned.grid!.height).toBeLessThan(reference.grid!.height);
+
+  // error: the reason, and no pinned rows.
+  await page.getByTestId("status-error").click();
+  await expect(statsCell).toHaveAttribute("data-stats-status", "error");
+  await expect(statsCell).toContainText("Stats error: stats_failed");
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual([]);
+  expect((await staysPut()).grid).toEqual(withoutPinned.grid);
+
+  // complete: the values are in the pinned area where the placeholders were, so
+  // the grid is as tall as it was while pending.
+  await page.getByTestId("status-complete").click();
+  await expect(statsCell).toHaveAttribute("data-stats-status", "complete");
+  await expect(statsCell).toContainText("Summary stats ready");
+  await expect.poll(distinctPinnedRowIds, { timeout: 10_000 }).toEqual(["main-dtype", "main-mean"]);
+  await expect(page.locator('.ag-floating-top .ag-cell[col-id="a"]').first()).toHaveText("int64");
+  expect(await boxes(page)).toEqual(reference);
+
+  expect(pageErrors).toEqual([]);
+});
+
+test("the Compute summary stats control sends stats_request with force", async ({ page }) => {
+  await page.goto(STORY_URL);
+  await waitForCells(page);
+
+  await page.getByTestId("status-not_computed").click();
+  await page.getByRole("button", { name: "Compute summary stats" }).click();
+
+  const sent = page.getByTestId("sent-log");
+  await expect.poll(async () => JSON.parse((await sent.textContent()) ?? "[]")).toEqual([
+    { type: "stats_request", stats_gen: 7, scope: "raw", force: true },
+  ]);
+});
+
+test("no control is offered while the stats are pending, in error or complete", async ({ page }) => {
+  await page.goto(STORY_URL);
+  await waitForCells(page);
+
+  const control = page.getByRole("button", { name: "Compute summary stats" });
+  for (const status of ["pending", "error", "complete"]) {
+    await page.getByTestId(`status-${status}`).click();
+    await expect(page.getByTestId("stats-status")).toHaveAttribute("data-stats-status", status);
+    await expect(control).toHaveCount(0);
+  }
+});